fix(tokenizer): 更新特殊token定义以支持新的对话格式

2025-10-25 16:50:41 +08:00
parent 491b39b23f
commit 1deb6d59a3
2 changed files with 9 additions and 22 deletions
@@ -8,7 +8,7 @@ from typing import List, Union
 class BpeTokenizer:
    def __init__(self, path=None):
        self._control_tokens = ["<bos>", "<eos>", "<pad>"]
-        self._special_tokens = ["<|user|>", "<|system|>"]
+        self._special_tokens = ["<|im_start|>", "<|im_end|>"]
        model = BPE()
        tokenizer = Tokenizer(model)
        tokenizer.normalizer = normalizers.Sequence([