refactor: 修改模型架构

2026-04-16 18:57:21 +08:00
parent cb93f8219e
commit a38334f4ce
17 changed files with 540 additions and 176 deletions
@@ -115,9 +115,9 @@ export_dataset(
 ### 2. 分词并缓存为 HDF5

 ```python
-from pipeline import BpeTokenizer, ProcessorFactory, cache_jsonl
+from pipeline import AutoTokenizer, ProcessorFactory, cache_jsonl

-tokenizer = BpeTokenizer("tokenizer.json")
+tokenizer = AutoTokenizer("tokenizer.json")
 processor = ProcessorFactory.create("pt", tokenizer)

 cache_jsonl(