refactor: 修改模型架构

pipeline/strategies/alpaca.py

@@ -2,7 +2,7 @@
 
 from typing import List
 
-from pipeline.tokenizer import BpeTokenizer
+from pipeline.tokenize import AutoTokenizer
 from pipeline.strategies.base import PromptStrategy
 from pipeline.strategies.factory import StrategyFactory
 
@@ -13,7 +13,7 @@ class AlpacaStrategy(PromptStrategy):
 
     def __init__(
         self,
-        tokenizer: BpeTokenizer,
+        tokenizer: AutoTokenizer,
         instruction_start: str = "### Instruction:\n",
         response_start: str = "### Response:\n",
         response_suffix: str = "\n<｜end▁of▁sentence｜>",

pipeline/strategies/base.py

@@ -3,7 +3,7 @@
 from abc import ABC, abstractmethod
 from typing import List
 
-from pipeline.tokenizer import BpeTokenizer
+from pipeline.tokenize import AutoTokenizer
 
 
 class PromptStrategy(ABC):
@@ -14,7 +14,7 @@ class PromptStrategy(ABC):
     which assembles them with pre-encoded format tokens.
     """
 
-    def __init__(self, tokenizer: BpeTokenizer):
+    def __init__(self, tokenizer: AutoTokenizer):
         self.tokenizer = tokenizer
 
     def _encode_format(self, text: str) -> List[int]:

pipeline/strategies/chatml.py

@@ -2,7 +2,7 @@
 
 from typing import List
 
-from pipeline.tokenizer import BpeTokenizer
+from pipeline.tokenize import AutoTokenizer
 from pipeline.strategies.base import PromptStrategy
 from pipeline.strategies.factory import StrategyFactory
 
@@ -13,7 +13,7 @@ class ChatMLStrategy(PromptStrategy):
 
     def __init__(
         self,
-        tokenizer: BpeTokenizer,
+        tokenizer: AutoTokenizer,
         user_start: str = "<｜im▁start｜>user\n",
         user_end: str = "<｜im▁end｜>\n",
         assistant_start: str = "<｜im▁start｜>assistant\n",

pipeline/strategies/factory.py

@@ -2,7 +2,7 @@
 
 from typing import Dict, List, Type
 
-from pipeline.tokenizer import BpeTokenizer
+from pipeline.tokenize import AutoTokenizer
 from pipeline.strategies.base import PromptStrategy
 
 
@@ -44,7 +44,7 @@ class StrategyFactory:
         return decorator
 
     @classmethod
-    def create(cls, name: str, tokenizer: BpeTokenizer, **kwargs) -> PromptStrategy:
+    def create(cls, name: str, tokenizer: AutoTokenizer, **kwargs) -> PromptStrategy:
         """Create a strategy by name.
 
         Args: