chore: 重命名目录
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
from astrai.tokenize.tokenizer import (
|
||||
BaseTokenizer,
|
||||
BpeTokenizer,
|
||||
BaseTrainer,
|
||||
BpeTrainer,
|
||||
)
|
||||
from astrai.tokenize.chat_template import (
|
||||
HistoryType,
|
||||
MessageType,
|
||||
build_prompt,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"BaseTokenizer",
|
||||
"BpeTokenizer",
|
||||
"BaseTrainer",
|
||||
"BpeTrainer",
|
||||
"HistoryType",
|
||||
"MessageType",
|
||||
"CHAT_TEMPLATES",
|
||||
"build_prompt",
|
||||
]
|
||||
@@ -0,0 +1,287 @@
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from jinja2 import Template
|
||||
from dataclasses import dataclass
|
||||
from astrai.factory import Registry
|
||||
|
||||
HistoryType = List[Tuple[str, str]]
|
||||
MessageType = Dict[str, str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatTemplate:
|
||||
"""A chat template with Jinja2 rendering support.
|
||||
|
||||
Attributes:
|
||||
name: Unique identifier for the template.
|
||||
template_str: Jinja2 template string.
|
||||
description: Optional description.
|
||||
default_variables: Optional dictionary of default variable values
|
||||
that will be passed to the template if not overridden during rendering.
|
||||
special_tokens: Optional dictionary mapping token names to their string values.
|
||||
These tokens are automatically added to the template variables.
|
||||
"""
|
||||
|
||||
name: str
|
||||
template_str: str
|
||||
description: str = ""
|
||||
default_variables: Dict[str, Any] = None
|
||||
special_tokens: Dict[str, str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.default_variables is None:
|
||||
self.default_variables = {}
|
||||
if self.special_tokens is None:
|
||||
self.special_tokens = {}
|
||||
|
||||
@classmethod
|
||||
def from_string(
|
||||
cls,
|
||||
template_str: str,
|
||||
description: str = "",
|
||||
default_variables: Optional[Dict[str, Any]] = None,
|
||||
special_tokens: Optional[Dict[str, str]] = None,
|
||||
) -> "ChatTemplate":
|
||||
"""Create a ChatTemplate instance directly from a template string."""
|
||||
return cls(
|
||||
name="", # empty name for ad‑hoc templates
|
||||
template_str=template_str,
|
||||
description=description,
|
||||
default_variables=default_variables,
|
||||
special_tokens=special_tokens,
|
||||
)
|
||||
|
||||
def render(
|
||||
self,
|
||||
messages: List[MessageType],
|
||||
system_prompt: Optional[str] = None,
|
||||
**extra_variables: Any,
|
||||
) -> str:
|
||||
"""Render the template with given messages and variables.
|
||||
|
||||
Args:
|
||||
messages: List of message dicts with 'role' and 'content'.
|
||||
system_prompt: Optional system prompt string.
|
||||
**extra_variables: Additional variables to pass to the template.
|
||||
These override default_variables and special_tokens.
|
||||
|
||||
Returns:
|
||||
Rendered prompt string.
|
||||
"""
|
||||
# Merge default variables, special tokens, and extra variables
|
||||
variables = {**self.default_variables, **self.special_tokens, **extra_variables}
|
||||
variables["messages"] = messages
|
||||
if system_prompt is not None:
|
||||
variables["system_prompt"] = system_prompt
|
||||
|
||||
jinja_template = Template(self.template_str)
|
||||
return jinja_template.render(**variables)
|
||||
|
||||
|
||||
# Global registry instance
|
||||
_default_registry = Registry()
|
||||
|
||||
# Default template name
|
||||
_default_template_name = "chatml"
|
||||
|
||||
|
||||
# Convenience functions
|
||||
def register_chat_template(
|
||||
name: str,
|
||||
template_str: str,
|
||||
description: str = "",
|
||||
default_variables: Optional[Dict[str, Any]] = None,
|
||||
special_tokens: Optional[Dict[str, str]] = None,
|
||||
) -> ChatTemplate:
|
||||
"""Register a chat template in the global registry."""
|
||||
template = ChatTemplate(
|
||||
name=name,
|
||||
template_str=template_str,
|
||||
description=description,
|
||||
default_variables=default_variables,
|
||||
special_tokens=special_tokens,
|
||||
)
|
||||
_default_registry.register(name, template, category=None, priority=0)
|
||||
return template
|
||||
|
||||
|
||||
def set_default_chat_template(name: str) -> None:
|
||||
"""Set the default chat template name globally."""
|
||||
global _default_template_name
|
||||
if not _default_registry.contains(name):
|
||||
raise KeyError(
|
||||
f"Chat template '{name}' not found. Available: {list(_default_registry.list_names())}"
|
||||
)
|
||||
_default_template_name = name
|
||||
|
||||
|
||||
def get_default_chat_template_name() -> str:
|
||||
"""Get the current default chat template name."""
|
||||
return _default_template_name
|
||||
|
||||
|
||||
def get_chat_template(name: str) -> ChatTemplate:
|
||||
"""Get a chat template from the global registry."""
|
||||
return _default_registry.get(name)
|
||||
|
||||
|
||||
def list_chat_templates() -> List[str]:
|
||||
"""List all registered chat template names."""
|
||||
return _default_registry.list_names()
|
||||
|
||||
|
||||
def chat_template_exists(name: str) -> bool:
|
||||
"""Check if a chat template exists."""
|
||||
return _default_registry.contains(name)
|
||||
|
||||
|
||||
def build_prompt(
|
||||
query: str,
|
||||
system_prompt: Optional[str] = None,
|
||||
history: Optional[HistoryType] = None,
|
||||
template: Optional[str] = None,
|
||||
template_name: Optional[str] = None,
|
||||
**extra_variables: Any,
|
||||
) -> str:
|
||||
"""Build prompt using a registered chat template or a custom template string.
|
||||
|
||||
This function maintains backward compatibility with the previous API.
|
||||
|
||||
Args:
|
||||
query: The current user query.
|
||||
system_prompt: Optional system prompt.
|
||||
history: Optional list of (user_msg, assistant_msg) pairs.
|
||||
template: If provided, uses this exact Jinja2 template string (overrides template_name).
|
||||
template_name: Name of a registered template to use (ignored if `template` is given).
|
||||
If None, uses the globally set default template (see `set_default_chat_template`).
|
||||
**extra_variables: Additional variables to pass to the template.
|
||||
|
||||
Returns:
|
||||
Rendered prompt string.
|
||||
|
||||
Raises:
|
||||
KeyError: If `template_name` is not registered.
|
||||
"""
|
||||
# Convert history to message format
|
||||
messages: List[MessageType] = []
|
||||
if history:
|
||||
for user_msg, assistant_msg in history:
|
||||
messages.append({"role": "user", "content": user_msg})
|
||||
messages.append({"role": "assistant", "content": assistant_msg})
|
||||
messages.append({"role": "user", "content": query})
|
||||
|
||||
if template is not None:
|
||||
# Use the provided template string directly
|
||||
jinja_template = Template(template)
|
||||
variables = {"messages": messages, **extra_variables}
|
||||
if system_prompt is not None:
|
||||
variables["system_prompt"] = system_prompt
|
||||
return jinja_template.render(**variables)
|
||||
else:
|
||||
# Determine which template name to use
|
||||
if template_name is None:
|
||||
template_name = _default_template_name
|
||||
# Use a registered template
|
||||
chat_template = get_chat_template(template_name)
|
||||
return chat_template.render(
|
||||
messages=messages,
|
||||
system_prompt=system_prompt,
|
||||
**extra_variables,
|
||||
)
|
||||
|
||||
|
||||
# Predefined templates
|
||||
# ChatML template (original)
|
||||
register_chat_template(
|
||||
name="chatml",
|
||||
template_str=(
|
||||
"{%- if system_prompt -%}\n"
|
||||
"{{ bos_token }}system\n"
|
||||
"{{ system_prompt }}{{ eos_token }}\n"
|
||||
"{%- endif -%}\n"
|
||||
"{%- for message in messages -%}\n"
|
||||
"{{ bos_token }}{{ message['role'] }}\n"
|
||||
"{{ message['content'] }}{{ eos_token }}\n"
|
||||
"{%- endfor -%}\n"
|
||||
"{{ bos_token }}assistant\n"
|
||||
),
|
||||
description="ChatML format with configurable special tokens.",
|
||||
special_tokens={"bos_token": "<｜im▁start｜>", "eos_token": "<｜im▁end｜>"},
|
||||
)
|
||||
|
||||
# Simplified template without special tokens (plain text)
|
||||
register_chat_template(
|
||||
name="plain",
|
||||
template_str=(
|
||||
"{%- if system_prompt -%}\n"
|
||||
"System: {{ system_prompt }}\n"
|
||||
"{%- endif -%}\n"
|
||||
"{%- for message in messages -%}\n"
|
||||
"{{ message['role']|capitalize }}: {{ message['content'] }}\n"
|
||||
"{%- endfor -%}\n"
|
||||
"Assistant:"
|
||||
),
|
||||
description="Plain text format with role labels.",
|
||||
)
|
||||
|
||||
# Alpaca-style template
|
||||
register_chat_template(
|
||||
name="alpaca",
|
||||
template_str=(
|
||||
"{%- if system_prompt -%}\n"
|
||||
"### Instruction:\n"
|
||||
"{{ system_prompt }}\n"
|
||||
"{%- endif -%}\n"
|
||||
"### Input:\n"
|
||||
"{{ messages[-1]['content'] }}\n"
|
||||
"### Response:"
|
||||
),
|
||||
description="Alpaca instruction‑response format (single‑turn).",
|
||||
default_variables={},
|
||||
)
|
||||
|
||||
# OpenAI chat format (approximation)
|
||||
register_chat_template(
|
||||
name="openai",
|
||||
template_str=(
|
||||
"{%- if system_prompt -%}\n"
|
||||
"{{ bos_token }}system\n"
|
||||
"{{ system_prompt }}{{ eos_token }}\n"
|
||||
"{%- endif -%}\n"
|
||||
"{%- for message in messages -%}\n"
|
||||
"{{ bos_token }}{{ message['role'] }}\n"
|
||||
"{{ message['content'] }}{{ eos_token }}\n"
|
||||
"{%- endfor -%}\n"
|
||||
"{{ bos_token }}assistant\n"
|
||||
),
|
||||
description="OpenAI‑compatible chat format with configurable special tokens.",
|
||||
special_tokens={"bos_token": "<｜im▁start｜>", "eos_token": "<｜im▁end｜>"},
|
||||
)
|
||||
|
||||
# Llama‑2 style with [INST] tags
|
||||
register_chat_template(
|
||||
name="llama2",
|
||||
template_str=(
|
||||
"{%- if system_prompt -%}\n"
|
||||
"<<SYS>>\n"
|
||||
"{{ system_prompt }}\n"
|
||||
"<</SYS>>\n"
|
||||
"{%- endif -%}\n"
|
||||
"[INST] {{ messages[-1]['content'] }} [/INST]"
|
||||
),
|
||||
description="Llama‑2 style with [INST] tags (single‑turn).",
|
||||
default_variables={},
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ChatTemplate",
|
||||
"register_chat_template",
|
||||
"get_chat_template",
|
||||
"list_chat_templates",
|
||||
"chat_template_exists",
|
||||
"build_prompt",
|
||||
"set_default_chat_template",
|
||||
"get_default_chat_template_name",
|
||||
"HistoryType",
|
||||
"MessageType",
|
||||
]
|
||||
@@ -0,0 +1,212 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Union
|
||||
|
||||
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
|
||||
from tokenizers.models import BPE
|
||||
from tokenizers.trainers import BpeTrainer as BpeTrainerImpl
|
||||
|
||||
|
||||
class BaseTokenizer(ABC):
|
||||
@abstractmethod
|
||||
def _init_tokenizer(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save(self, path):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load(self, path):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def encode(
|
||||
self,
|
||||
tokens: Union[str, List[str]],
|
||||
out_ids: bool = True,
|
||||
add_special_tokens: bool = False,
|
||||
) -> List:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def decode(self, tokens: List[int], skip_special_tokens: bool = True) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def __len__(self) -> int:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def stop_ids(self) -> List[int]:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def bos_id(self) -> int:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def eos_id(self) -> int:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def pad_id(self) -> int:
|
||||
pass
|
||||
|
||||
|
||||
class BaseTrainer(ABC):
|
||||
def __init__(self, tokenizer: BaseTokenizer):
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
@abstractmethod
|
||||
def train(self, files, vocab_size, min_freq, **kwargs):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def train_from_iterator(self, iterator, vocab_size, min_freq, **kwargs):
|
||||
pass
|
||||
|
||||
|
||||
class BpeTokenizer(BaseTokenizer):
|
||||
def __init__(
|
||||
self,
|
||||
control_tokens: List[str] = None,
|
||||
special_tokens: List[str] = None,
|
||||
path=None,
|
||||
):
|
||||
self._control_tokens = control_tokens or [
|
||||
"<｜begin▁of▁sentence｜>",
|
||||
"<｜end▁of▁sentence｜>",
|
||||
"<｜▁pad▁｜>",
|
||||
]
|
||||
self._special_tokens = special_tokens or [
|
||||
"<｜im▁start｜>",
|
||||
"<｜im▁end｜>",
|
||||
]
|
||||
self._tokenizer = None
|
||||
self._init_tokenizer()
|
||||
if path is not None:
|
||||
self.load(path)
|
||||
|
||||
def _init_tokenizer(self):
|
||||
model = BPE()
|
||||
self._tokenizer = Tokenizer(model)
|
||||
self._tokenizer.normalizer = normalizers.Sequence(
|
||||
[normalizers.NFC(), normalizers.Strip()]
|
||||
)
|
||||
self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
|
||||
[
|
||||
pre_tokenizers.UnicodeScripts(),
|
||||
pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=True),
|
||||
]
|
||||
)
|
||||
self._tokenizer.decoder = decoders.ByteLevel()
|
||||
self._tokenizer.post_processor = processors.ByteLevel(trim_offsets=True)
|
||||
|
||||
def save(self, path):
|
||||
self._tokenizer.save(path)
|
||||
|
||||
def load(self, path):
|
||||
self._tokenizer = Tokenizer.from_file(path)
|
||||
|
||||
def encode(
|
||||
self,
|
||||
tokens: Union[str, List[str]],
|
||||
out_ids: bool = True,
|
||||
add_special_tokens: bool = False,
|
||||
) -> List:
|
||||
if isinstance(tokens, str):
|
||||
encoded = self._tokenizer.encode(
|
||||
tokens, add_special_tokens=add_special_tokens
|
||||
)
|
||||
return encoded.ids if out_ids else encoded.tokens
|
||||
else:
|
||||
encoded_list = self._tokenizer.encode_batch(
|
||||
tokens, add_special_tokens=add_special_tokens
|
||||
)
|
||||
return [
|
||||
encoded.ids if out_ids else encoded.tokens for encoded in encoded_list
|
||||
]
|
||||
|
||||
def decode(self, tokens: List[int], skip_special_tokens: bool = True) -> str:
|
||||
return self._tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self._tokenizer.get_vocab_size()
|
||||
|
||||
@property
|
||||
def stop_ids(self) -> List[int]:
|
||||
stop_token = self._control_tokens + self._special_tokens
|
||||
return [self._tokenizer.token_to_id(tok) for tok in stop_token]
|
||||
|
||||
@property
|
||||
def bos_id(self) -> int:
|
||||
return self._tokenizer.token_to_id(self._control_tokens[0])
|
||||
|
||||
@property
|
||||
def eos_id(self) -> int:
|
||||
return self._tokenizer.token_to_id(self._control_tokens[1])
|
||||
|
||||
@property
|
||||
def pad_id(self) -> int:
|
||||
return self._tokenizer.token_to_id(self._control_tokens[2])
|
||||
|
||||
|
||||
class BpeTrainer(BaseTrainer):
|
||||
def __init__(self, tokenizer: BaseTokenizer):
|
||||
super().__init__(tokenizer)
|
||||
|
||||
def _prepare_trainer(
|
||||
self,
|
||||
vocab_size: int,
|
||||
min_freq: int,
|
||||
reserved_token_size: int,
|
||||
max_token_length=18,
|
||||
):
|
||||
assert reserved_token_size > len(self.tokenizer._special_tokens)
|
||||
reserved_tokens = [
|
||||
f"<|reserve{i:02d}|>"
|
||||
for i in range(reserved_token_size - len(self.tokenizer._special_tokens))
|
||||
]
|
||||
detail_vocab_size = vocab_size - (
|
||||
len(reserved_tokens) + len(self.tokenizer._special_tokens)
|
||||
)
|
||||
alphabet = pre_tokenizers.ByteLevel.alphabet()
|
||||
min_size = len(alphabet) + len(self.tokenizer._control_tokens)
|
||||
assert detail_vocab_size > min_size
|
||||
|
||||
trainer = BpeTrainerImpl(
|
||||
vocab_size=detail_vocab_size,
|
||||
min_frequency=min_freq,
|
||||
limit_alphabet=detail_vocab_size // 6,
|
||||
max_token_length=max_token_length,
|
||||
special_tokens=self.tokenizer._control_tokens,
|
||||
initial_alphabet=alphabet,
|
||||
show_progress=True,
|
||||
)
|
||||
return trainer, reserved_tokens
|
||||
|
||||
def train(self, files, vocab_size, min_freq, reserved_token_size=100, **kwargs):
|
||||
trainer, reserved_tokens = self._prepare_trainer(
|
||||
vocab_size, min_freq, reserved_token_size, **kwargs
|
||||
)
|
||||
self.tokenizer._tokenizer.train(files=files, trainer=trainer)
|
||||
self.tokenizer._tokenizer.add_special_tokens(
|
||||
self.tokenizer._special_tokens + reserved_tokens
|
||||
)
|
||||
|
||||
def train_from_iterator(
|
||||
self, iterator, vocab_size, min_freq, reserved_token_size=100, **kwargs
|
||||
):
|
||||
trainer, reserved_tokens = self._prepare_trainer(
|
||||
vocab_size, min_freq, reserved_token_size, **kwargs
|
||||
)
|
||||
self.tokenizer._tokenizer.train_from_iterator(
|
||||
iterator=iterator, trainer=trainer
|
||||
)
|
||||
self.tokenizer._tokenizer.add_special_tokens(
|
||||
self.tokenizer._special_tokens + reserved_tokens
|
||||
)
|
||||
Reference in New Issue
Block a user