refactor: 修改模型架构

2026-04-16 18:57:21 +08:00
parent cb93f8219e
commit a38334f4ce
17 changed files with 540 additions and 176 deletions
@@ -0,0 +1,12 @@
+"""
+Tokenizer module with BPE implementation and auto-loading support.
+"""
+
+from pipeline.tokenize.tokenizer import AutoTokenizer, train_bpe_tokenizer
+from pipeline.tokenize.chat_template import ChatTemplate
+
+__all__ = [
+    "AutoTokenizer",
+    "train_bpe_tokenizer",
+    "ChatTemplate",
+]