refactor: 重构流水线架构，添加Pipeline抽象并拆分IOHandler

pipeline/__init__.py

@@ -1,10 +1,50 @@
-import logging
+"""DataPipeline: A flexible data processing pipeline for LLM training.
+
+Architecture:
+    - Pipeline: Composable stage-based processing
+    - Processors: Data transformation (pretrain, sft, dpo)
+    - Strategies: Prompt format abstraction (ChatML, Alpaca)
+    - I/O: File scanning and HDF5 storage
+
+Usage::
+
+    from pipeline import Pipeline, ProcessorFactory, FileScanner, HDF5Handler
+    from pipeline.pipeline import TransformStage
+    from pipeline.io import export_dataset, cache_jsonl
+
+    # Create pipeline
+    pipeline = Pipeline()
+    pipeline.add_stages(
+        TransformStage("normalize", normalizer.normalize),
+        TransformStage("tokenize", tokenizer.encode),
+    )
+
+    # Process data
+    results = pipeline.run(texts)
+    HDF5Handler.save("./output", "data", {"tokens": results})
+"""
+
+# Core modules
+from pipeline.pipeline import Pipeline, PipelineConfig, Stage, TransformStage
 from pipeline.tokenize import AutoTokenizer, ChatTemplate, train_bpe_tokenizer
 from pipeline.text import TextNormalizer
 from pipeline.packing import SequencePacker
-from pipeline.io import IOHandler, export_dataset, cache_jsonl
-from pipeline.processors import ProcessorFactory, BaseProcessor
-from pipeline.utils import setup_logging
+
+# I/O module
+from pipeline.io import FileScanner, HDF5Handler, export_dataset, cache_jsonl
+
+# Processors
+from pipeline.processors import (
+    ProcessorFactory,
+    BaseProcessor,
+    ProcessorSchema,
+    ProcessorConfig,
+    PreTrainProcessor,
+    SFTProcessor,
+    DPOProcessor,
+)
+
+# Strategies
 from pipeline.strategies import (
     PromptStrategy,
     ChatMLStrategy,
@@ -12,25 +52,43 @@ from pipeline.strategies import (
     StrategyFactory,
 )
 
-# Configure project-level logging
-setup_logging()
+# Utilities (lazy initialization)
+from pipeline import utils
+
+# Expose setup_logging for explicit use
+setup_logging = utils.setup_logging
 
 __all__ = [
+    # Pipeline
+    "Pipeline",
+    "PipelineConfig",
+    "Stage",
+    "TransformStage",
     # Tokenizer
     "AutoTokenizer",
     "ChatTemplate",
     "train_bpe_tokenizer",
-    # Core modules
+    # Text processing
     "TextNormalizer",
     "SequencePacker",
-    "IOHandler",
-    "ProcessorFactory",
-    "BaseProcessor",
+    # I/O
+    "FileScanner",
+    "HDF5Handler",
     "export_dataset",
     "cache_jsonl",
-    # Strategy pattern
+    # Processors
+    "ProcessorFactory",
+    "BaseProcessor",
+    "ProcessorSchema",
+    "ProcessorConfig",
+    "PreTrainProcessor",
+    "SFTProcessor",
+    "DPOProcessor",
+    # Strategies
     "PromptStrategy",
     "ChatMLStrategy",
     "AlpacaStrategy",
     "StrategyFactory",
+    # Utils
+    "setup_logging",
 ]