refactor : 基于声明式 JSON 配置的预处理管线重构

- 用工厂注册的 MaskBuilder（chat/instruction/text）替换硬编码的 _transform_* 方法
- mask 规则以 role-to-action 映射声明在配置中，与 chat_template 完全解耦
- 单次编码 + role-span 追踪替代两次编码 + 长度差计算 mask 的方式
- 支持多轮对话训练：所有 assistant 轮次参与训练，而非仅最后一轮
- 新建 astrai.preprocessing 包（builder.py + pipeline.py），删除 astrai/preprocess.py
- CLI 精简为 --config 参数，所有参数通过 PipelineConfig JSON 配置
- 新增 PipelineConfig、InputConfig、ProcessingConfig、OutputConfig dataclass
- 文档：assets/docs/preprocessing.md
- 27 个测试覆盖 mask builder、pipeline、配置序列化、工厂注册

scripts/tools/preprocess.py

@@ -1,108 +1,36 @@
-"""CLI: raw JSONL → tokenized .h5/.bin via Pipeline."""
+"""CLI: JSONL → tokenized .h5/.bin via config-driven Pipeline."""
 
 import argparse
-import sys
 
-from astrai.preprocess import Pipeline, detect_format
+from astrai.config.preprocess_config import PipelineConfig
+from astrai.preprocessing.pipeline import Pipeline
 
 
 def main():
     parser = argparse.ArgumentParser(
-        description="Raw JSONL → tokenized .h5/.bin for training"
+        description="Raw JSONL → tokenized .h5/.bin via config-driven Pipeline"
     )
     parser.add_argument(
         "inputs", nargs="+", metavar="JSONL", help="One or more JSONL files"
     )
+    parser.add_argument("--output_dir", "-o", required=True, help="Output directory")
     parser.add_argument(
-        "--output_dir",
-        "-o",
-        required=True,
-        help="Output directory (domain subdirs auto-created)",
+        "--config", "-c", required=True, help="Path to pipeline config JSON"
     )
     parser.add_argument(
         "--tokenizer_path",
         default="params",
-        help="Path to tokenizer (default: params)",
-    )
-    parser.add_argument(
-        "--text_key",
-        default=None,
-        help="JSON key for text (auto-detect if omitted)",
-    )
-    parser.add_argument(
-        "--domain_key",
-        default=None,
-        help="JSON key for domain label (auto-detect if omitted)",
-    )
-    parser.add_argument(
-        "--max_len",
-        type=int,
-        default=2048,
-        help="Max token length per doc (default: 2048)",
-    )
-    parser.add_argument(
-        "--min_text_len",
-        type=int,
-        default=50,
-        help="Min chars per doc (default: 50)",
-    )
-    parser.add_argument(
-        "--max_text_len",
-        type=int,
-        default=2_000_000,
-        help="Max chars per doc (default: 2000000)",
-    )
-    parser.add_argument(
-        "--no_dedup",
-        action="store_true",
-        help="Skip exact dedup",
-    )
-    parser.add_argument(
-        "--max_items",
-        type=int,
-        default=None,
-        help="Max docs to process (default: all)",
-    )
-    parser.add_argument(
-        "--max_tokens_per_shard",
-        type=int,
-        default=100_000_000,
-        help="Max tokens per .h5 shard (default: 100M)",
-    )
-    parser.add_argument(
-        "--format",
-        dest="storage_format",
-        choices=["h5", "bin"],
-        default="bin",
-        help="Output format (default: bin)",
-    )
-    parser.add_argument(
-        "--detect",
-        action="store_true",
-        help="Detect and print JSONL schema, then exit",
+        help="Path to tokenizer directory (default: params)",
     )
     args = parser.parse_args()
 
-    if args.detect:
-        fmt = detect_format(args.inputs)
-        print(f"text key   : {fmt['text_key']}")
-        print(f"domain key : {fmt['domain_key']}")
-        print(f"chat mode  : {fmt['is_chat']}")
-        sys.exit(0)
+    config = PipelineConfig.from_json(args.config)
 
     Pipeline(
+        config=config,
         input_paths=args.inputs,
         output_dir=args.output_dir,
         tokenizer_path=args.tokenizer_path,
-        text_key=args.text_key,
-        domain_key=args.domain_key,
-        max_len=args.max_len,
-        min_text_len=args.min_text_len,
-        max_text_len=args.max_text_len,
-        dedup=not args.no_dedup,
-        max_items=args.max_items,
-        max_tokens_per_shard=args.max_tokens_per_shard,
-        storage_format=args.storage_format,
     ).run()