refactor: 修改模型架构

2026-04-16 18:57:21 +08:00
parent cb93f8219e
commit a38334f4ce
17 changed files with 540 additions and 176 deletions
@@ -2,7 +2,7 @@

 from typing import List

-from pipeline.tokenizer import BpeTokenizer
+from pipeline.tokenize import AutoTokenizer
 from pipeline.strategies.base import PromptStrategy
 from pipeline.strategies.factory import StrategyFactory

@@ -13,7 +13,7 @@ class ChatMLStrategy(PromptStrategy):

    def __init__(
        self,
-        tokenizer: BpeTokenizer,
+        tokenizer: AutoTokenizer,
        user_start: str = "<｜im▁start｜>user\n",
        user_end: str = "<｜im▁end｜>\n",
        assistant_start: str = "<｜im▁start｜>assistant\n",