fix: 修复 nl_id + BFD 按 group_size 分批打包（存盘不拆分文件）

2026-07-03 17:07:26 +08:00
parent 598e1ce4ae
commit 06735b9cb3
2 changed files with 44 additions and 19 deletions
@@ -20,7 +20,7 @@ class ChatMLStrategy(PromptStrategy):
        assistant_end: str = "<｜im▁end｜>",
    ):
        super().__init__(tokenizer)
-        nl_id = tokenizer.token_to_id("\n")
+        nl_id = tokenizer.encode("a\nb", add_special_tokens=False)[1]

        self._user_start_ids = self._encode_format(user_start) + [nl_id]
        self._user_end_ids = self._encode_format(user_end) + [nl_id]