fix: 修复特殊token 问题

pipeline/strategies/__init__.py

@@ -1,4 +1,5 @@
 """Strategy pattern for prompt/response format abstraction."""
+
 from pipeline.strategies.base import PromptStrategy
 from pipeline.strategies.factory import StrategyFactory
 

pipeline/strategies/alpaca.py

@@ -1,4 +1,5 @@
 """Alpaca format strategy."""
+
 from typing import List
 
 from pipeline.tokenizer import BpeTokenizer
@@ -8,14 +9,14 @@ from pipeline.strategies.factory import StrategyFactory
 
 @StrategyFactory.register("alpaca")
 class AlpacaStrategy(PromptStrategy):
-    """Alpaca format: ``### Instruction: ... \\n\\n### Response: ... <eos>``"""
+    """Alpaca format:"""
 
     def __init__(
         self,
         tokenizer: BpeTokenizer,
         instruction_start: str = "### Instruction:\n",
         response_start: str = "### Response:\n",
-        response_suffix: str = "\n<eos>",
+        response_suffix: str = "\n<｜end▁of▁sentence｜>",
     ):
         super().__init__(tokenizer)
         self.instruction_start = instruction_start

pipeline/strategies/base.py

@@ -1,4 +1,5 @@
 """Abstract base class for prompt construction strategies."""
+
 from abc import ABC, abstractmethod
 from typing import List
 
@@ -30,7 +31,7 @@ class PromptStrategy(ABC):
         """Assemble query tokens into a complete prompt with format tokens.
 
         The prompt includes all tokens up to (and including) the response
-        start marker, e.g. ``<|im_start|>assistant\n``.
+        start marker, e.g. ``<｜im▁start｜>assistant\n``.
         """
 
     @abstractmethod

pipeline/strategies/chatml.py

@@ -1,4 +1,5 @@
 """ChatML format strategy."""
+
 from typing import List
 
 from pipeline.tokenizer import BpeTokenizer
@@ -8,15 +9,15 @@ from pipeline.strategies.factory import StrategyFactory
 
 @StrategyFactory.register("chatml")
 class ChatMLStrategy(PromptStrategy):
-    """ChatML format: ``<|im_start|>user ... <|im_end|> <|im_start|>assistant ... <|im_end|> <eos>``"""
+    """ChatML format strategy."""
 
     def __init__(
         self,
         tokenizer: BpeTokenizer,
-        user_start: str = "<|im_start|>user\n",
-        user_end: str = "<|im_end|>\n",
-        assistant_start: str = "<|im_start|>assistant\n",
-        assistant_end: str = "<|im_end|>\n<eos>",
+        user_start: str = "<｜im▁start｜>user\n",
+        user_end: str = "<｜im▁end｜>\n",
+        assistant_start: str = "<｜im▁start｜>assistant\n",
+        assistant_end: str = "<｜im▁end｜>\n",
     ):
         super().__init__(tokenizer)
 

pipeline/strategies/factory.py

@@ -1,4 +1,5 @@
 """Factory for creating and registering prompt strategies."""
+
 from typing import Dict, List, Type
 
 from pipeline.tokenizer import BpeTokenizer