fix(module): 修复一些错误

modules/tokenizer.py

@@ -93,9 +93,7 @@ class BpeTokenizer:
     
     @property
     def stop_ids(self) -> List[int]:
-        stop_ids = []
+        stop_ids = self._control_tokens + self._special_tokens
-        for token in self._control_tokens:
-            stop_ids.append(self._tokenizer.token_to_id(token))
         return stop_ids
     
     @property
@@ -109,11 +107,3 @@ class BpeTokenizer:
     @property
     def pad_id(self) -> int:
         return self._tokenizer.token_to_id("<pad>")
-    
-    @property
-    def user_id(self) -> int:
-        return self._tokenizer.token_to_id("<|user|>")
-    
-    @property
-    def system_id(self) -> int:
-        return self._tokenizer.token_to_id("<|system|>")

modules/utils.py

@@ -119,8 +119,8 @@ def get_pt_processor(tokenizer: BpeTokenizer):
 def get_sft_processor(tokenizer: BpeTokenizer):
     def processor(input_dict: dict):
         query, response = input_dict["query"], input_dict["response"]
-        q = tokenizer.encode(f"<|im_start|>user\n{query}<|im_end|>\n<|im_start|>assistant")
+        q = tokenizer.encode(f"<|im_start|>user\n{query}<|im_end|>\n<|im_start|>assistant\n")
-        a = tokenizer.encode(f"\n{response}<im_end>\n<eos>")
+        a = tokenizer.encode(f"{response}<|im_end|>\n<eos>")
         tokens = torch.tensor(q + a, dtype=torch.int32)
         loss_mask = torch.zeros_like(tokens, dtype=torch.bool)
         loss_mask[len(q):] = True