fix(module): 修复一些错误

2025-10-30 15:17:06 +08:00
parent 1deb6d59a3
commit 397c8e5d0f
2 changed files with 4 additions and 14 deletions
@@ -93,9 +93,7 @@ class BpeTokenizer:
    
    @property
    def stop_ids(self) -> List[int]:
-        stop_ids = []
-        for token in self._control_tokens:
-            stop_ids.append(self._tokenizer.token_to_id(token))
+        stop_ids = self._control_tokens + self._special_tokens
        return stop_ids
    
    @property
@@ -108,12 +106,4 @@ class BpeTokenizer:
    
    @property
    def pad_id(self) -> int:
-        return self._tokenizer.token_to_id("<pad>")
-    
-    @property
-    def user_id(self) -> int:
-        return self._tokenizer.token_to_id("<|user|>")
-    
-    @property
-    def system_id(self) -> int:
-        return self._tokenizer.token_to_id("<|system|>")
+        return self._tokenizer.token_to_id("<pad>")