refactor: 重构流水线架构，添加Pipeline抽象并拆分IOHandler

2026-04-23 19:45:57 +08:00
parent a38334f4ce
commit cb6bfcb976
14 changed files with 886 additions and 184 deletions
@@ -0,0 +1,19 @@
+"""I/O module for file operations, HDF5 storage, and dataset export.
+
+This module provides:
+- FileScanner: File and directory scanning utilities
+- HDF5Handler: Tensor data persistence
+- export_dataset: HuggingFace Dataset to JSONL export
+- cache_jsonl: JSONL to HDF5 tokenization and caching
+"""
+
+from pipeline.io.file_scanner import FileScanner
+from pipeline.io.hdf5_handler import HDF5Handler
+from pipeline.io.export import export_dataset, cache_jsonl
+
+__all__ = [
+    "FileScanner",
+    "HDF5Handler",
+    "export_dataset",
+    "cache_jsonl",
+]