refactor: 重构流水线架构，添加Pipeline抽象并拆分IOHandler

2026-04-23 19:45:57 +08:00
parent a38334f4ce
commit cb6bfcb976
14 changed files with 886 additions and 184 deletions
@@ -4,15 +4,18 @@ Processor classes are registered at definition time via decorators and
 can be created through :class:`ProcessorFactory`.
 """

-from pipeline.processors.base import BaseProcessor
-from pipeline.processors.factory import ProcessorFactory
+from pipeline.processors.base import BaseProcessor, ProcessorSchema, encode_with_mask
+from pipeline.processors.factory import ProcessorFactory, ProcessorConfig
 from pipeline.processors.pretrain import PreTrainProcessor
 from pipeline.processors.sft import SFTProcessor
 from pipeline.processors.dpo import DPOProcessor

 __all__ = [
    "BaseProcessor",
+    "ProcessorSchema",
+    "ProcessorConfig",
    "ProcessorFactory",
+    "encode_with_mask",
    "PreTrainProcessor",
    "SFTProcessor",
    "DPOProcessor",