refactor: 重构数据管道模块

2026-03-20 21:36:37 +08:00
parent 7045de9c32
commit 3fb2326115
20 changed files with 739 additions and 345 deletions
@@ -0,0 +1,17 @@
+from .tokenizer import BpeTokenizer
+from .datapipeline import (
+    DataPipeline,
+    ProcessorFactory,
+    IOHandler,
+    TextNormalizer,
+    SequencePacker
+)
+
+__all__ = [
+    'BpeTokenizer',
+    'DataPipeline',
+    'ProcessorFactory',
+    'IOHandler',
+    'TextNormalizer',
+    'SequencePacker'
+]