chore: 修改类名,优化导入顺序
This commit is contained in:
@@ -1,23 +1,15 @@
|
||||
from astrai.tokenize.chat_template import ChatTemplate, MessageType
|
||||
from astrai.tokenize.tokenizer import (
|
||||
TextTokenizer,
|
||||
AutoTokenizer,
|
||||
BpeTokenizer,
|
||||
)
|
||||
from astrai.tokenize.trainer import BpeTrainer
|
||||
from astrai.tokenize.chat_template import (
|
||||
ChatTemplate,
|
||||
HistoryType,
|
||||
MessageType,
|
||||
)
|
||||
|
||||
# Alias for compatibility
|
||||
AutoTokenizer = TextTokenizer
|
||||
|
||||
__all__ = [
|
||||
"TextTokenizer",
|
||||
"AutoTokenizer",
|
||||
"BpeTokenizer",
|
||||
"BpeTrainer",
|
||||
"ChatTemplate",
|
||||
"HistoryType",
|
||||
"MessageType",
|
||||
"HistoryType",
|
||||
]
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from jinja2 import Template
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
HistoryType = List[Tuple[str, str]]
|
||||
MessageType = Dict[str, str]
|
||||
from jinja2 import Template
|
||||
|
||||
# Message type for chat messages
|
||||
type MessageType = Dict[str, Any]
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -8,10 +8,11 @@ from typing import Dict, List, Optional, Union
|
||||
|
||||
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
|
||||
from tokenizers.models import BPE
|
||||
|
||||
from astrai.tokenize.chat_template import ChatTemplate
|
||||
|
||||
|
||||
class TextTokenizer:
|
||||
class AutoTokenizer:
|
||||
"""Base tokenizer class with automatic loading support"""
|
||||
|
||||
TOKENIZER_CLASSES = {} # Registry for auto-loading
|
||||
@@ -51,7 +52,7 @@ class TextTokenizer:
|
||||
self.set_chat_template(config["chat_template"])
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, path: Union[str, Path], **kwargs) -> "TextTokenizer":
|
||||
def from_pretrained(cls, path: Union[str, Path], **kwargs) -> "AutoTokenizer":
|
||||
"""Load tokenizer from pretrained directory."""
|
||||
instance = cls(path)
|
||||
return instance
|
||||
@@ -235,7 +236,7 @@ class TextTokenizer:
|
||||
return rendered
|
||||
|
||||
|
||||
class BpeTokenizer(TextTokenizer):
|
||||
class BpeTokenizer(AutoTokenizer):
|
||||
"""BPE tokenizer implementation."""
|
||||
|
||||
def __init__(
|
||||
|
||||
Reference in New Issue
Block a user