refactor: 修改项目结构

2026-03-21 21:25:59 +08:00
parent 5b47ab1dea
commit 909f1b400c
32 changed files with 535 additions and 703 deletions
@@ -0,0 +1,18 @@
+from .tokenizer import BpeTokenizer
+from .text import TextNormalizer
+from .packing import SequencePacker
+from .io import IOHandler
+from .processors import ProcessorFactory, BaseProcessor
+from .export import export_dataset
+from .cache import cache_jsonl
+
+__all__ = [
+    'BpeTokenizer',
+    'TextNormalizer',
+    'SequencePacker',
+    'IOHandler',
+    'ProcessorFactory',
+    'BaseProcessor',
+    'export_dataset',
+    'cache_jsonl',
+]