feat: 实现模型动态注册机制

2026-04-05 19:38:12 +08:00
parent ff43a2fab8
commit fc278d17ab
25 changed files with 686 additions and 651 deletions
@@ -1,22 +1,23 @@
 from astrai.tokenize.tokenizer import (
-    BaseTokenizer,
+    TextTokenizer,
    BpeTokenizer,
-    BaseTrainer,
-    BpeTrainer,
 )
+from astrai.tokenize.trainer import BpeTrainer
 from astrai.tokenize.chat_template import (
+    ChatTemplate,
    HistoryType,
    MessageType,
-    build_prompt,
 )

+# Alias for compatibility
+AutoTokenizer = TextTokenizer
+
 __all__ = [
-    "BaseTokenizer",
+    "TextTokenizer",
+    "AutoTokenizer",
    "BpeTokenizer",
-    "BaseTrainer",
    "BpeTrainer",
+    "ChatTemplate",
    "HistoryType",
    "MessageType",
-    "CHAT_TEMPLATES",
-    "build_prompt",
 ]