feat(khaosz/core/tokenizer): 添加 user_id 和 system_id 属性

2025-09-29 13:47:37 +08:00
parent 89211c16f6
commit bdda1cc35a
2 changed files with 9 additions and 1 deletions
@@ -109,3 +109,11 @@ class BpeTokenizer:
    @property
    def pad_id(self) -> int:
        return self._tokenizer.token_to_id("<pad>")
+    
+    @property
+    def user_id(self) -> int:
+        return self._tokenizer.token_to_id("<|user|>")
+    
+    @property
+    def system_id(self) -> int:
+        return self._tokenizer.token_to_id("<|system|>")