fix: 修复特殊token 问题

2026-04-02 16:16:02 +08:00
parent e01ec081b3
commit f44ad6912e
28 changed files with 334 additions and 163 deletions
@@ -5,24 +5,29 @@ from pipeline.packing import SequencePacker
 from pipeline.io import IOHandler, export_dataset, cache_jsonl
 from pipeline.processors import ProcessorFactory, BaseProcessor
 from pipeline.utils import setup_logging
-from pipeline.strategies import PromptStrategy, ChatMLStrategy, AlpacaStrategy, StrategyFactory
+from pipeline.strategies import (
+    PromptStrategy,
+    ChatMLStrategy,
+    AlpacaStrategy,
+    StrategyFactory,
+)

 # Configure project-level logging
 setup_logging()

 __all__ = [
    # Core modules
-    'BpeTokenizer',
-    'TextNormalizer',
-    'SequencePacker',
-    'IOHandler',
-    'ProcessorFactory',
-    'BaseProcessor',
-    'export_dataset',
-    'cache_jsonl',
+    "BpeTokenizer",
+    "TextNormalizer",
+    "SequencePacker",
+    "IOHandler",
+    "ProcessorFactory",
+    "BaseProcessor",
+    "export_dataset",
+    "cache_jsonl",
    # Strategy pattern
-    'PromptStrategy',
-    'ChatMLStrategy',
-    'AlpacaStrategy',
-    'StrategyFactory',
+    "PromptStrategy",
+    "ChatMLStrategy",
+    "AlpacaStrategy",
+    "StrategyFactory",
 ]