fix: 修复特殊token 问题

pipeline/processors/__init__.py

@@ -3,6 +3,7 @@
 Processor classes are registered at definition time via decorators and
 can be created through :class:`ProcessorFactory`.
 """
+
 from pipeline.processors.base import BaseProcessor
 from pipeline.processors.factory import ProcessorFactory
 from pipeline.processors.pretrain import PreTrainProcessor

pipeline/processors/base.py

@@ -1,4 +1,5 @@
 """Processor base class and shared utilities."""
+
 from abc import ABC, abstractmethod
 from typing import Dict, List, Any, Tuple
 

pipeline/processors/dpo.py

@@ -1,4 +1,5 @@
 """DPO preference learning data processor."""
+
 from typing import Dict, List, Any, Optional
 
 import torch

pipeline/processors/factory.py

@@ -1,4 +1,5 @@
 """Factory for creating and registering processors."""
+
 from typing import Dict, List, Any, Optional, Type
 
 from pipeline.processors.base import BaseProcessor

pipeline/processors/pretrain.py

@@ -1,4 +1,5 @@
 """Pre-training data processor."""
+
 from typing import Dict, List, Any
 
 import torch
@@ -18,7 +19,7 @@ class PreTrainProcessor(BaseProcessor):
 
     def process(self, input_dict: Dict[str, Any]) -> Dict[str, Tensor]:
         segment = input_dict["text"]
-        tokens = self.tokenizer.encode(f"{segment}<eos>")
+        tokens = self.tokenizer.encode(f"{segment}<｜end▁of▁sentence｜>")
         return {"sequence": torch.tensor(tokens, dtype=torch.int32)}
 
     @property

pipeline/processors/sft.py

@@ -1,4 +1,5 @@
 """Supervised fine-tuning data processor."""
+
 from typing import Dict, List, Any, Optional
 
 import torch