feat: MinHash+LSH 去重 + Strategy/Factory 存储后端

scripts/cache_h5.py

@@ -1,11 +1,10 @@
-"""JSONL to H5 caching script.
+"""JSONL tokenization and caching script.
 
-Tokenize JSONL files and pack them into HDF5 format.
+Tokenize JSONL files and save as HDF5 or binary format.
 
 Usage:
     python scripts/cache_h5.py pt ./dataset/chinese-c4-pretrain
-    python scripts/cache_h5.py sft ./dataset/belle-sft --pack-size 4096 --strategy alpaca
-    python scripts/cache_h5.py sft ./dataset/Ling-Coder-sft --tokenizer ./my_tokenizer.json
+    python scripts/cache_h5.py sft ./dataset/belle-sft --pack-size 4096 --output-format bin
 """
 
 import argparse
@@ -29,7 +28,7 @@ def main():
         "-o",
         "--output-dir",
         default=None,
-        help="H5 output dir (default: <input_dir>/cached)",
+        help="Output dir (default: <input_dir>/cached)",
     )
     parser.add_argument(
         "-t",
@@ -73,6 +72,13 @@ def main():
         choices=["DEBUG", "INFO", "WARNING", "ERROR"],
         help="Logging level (default: INFO)",
     )
+    parser.add_argument(
+        "-f",
+        "--output-format",
+        default="h5",
+        choices=["h5", "bin"],
+        help="Output format: h5 or bin (default: h5)",
+    )
     args = parser.parse_args()
 
     # Initialize logging explicitly (not automatic anymore)
@@ -126,6 +132,7 @@ def main():
         pad_value=args.pad_value,
         group_size=args.group_size,
         pack_algo=args.pack_algo,
+        output_format=args.output_format,
     )
     print(f"\nDone! Output saved to {output_dir}")
 

scripts/dedup_pretrain.py

@@ -0,0 +1,36 @@
+"""MinHash + LSH deduplication CLI.
+
+Usage:
+    python scripts/dedup_pretrain.py --input-dir <data_dir> --output-dir <out_dir> --threshold 0.8 --num-perm 128 --output-format jsonl
+"""
+
+import argparse
+
+from pipeline.io import dedup_jsonl
+
+
+def main():
+    parser = argparse.ArgumentParser(description="MinHash + LSH deduplication")
+    parser.add_argument("--input-dir", required=True)
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument("--threshold", type=float, default=0.8)
+    parser.add_argument("--num-perm", type=int, default=128)
+    parser.add_argument("--ngram", type=int, default=3)
+    parser.add_argument("--output-format", default="jsonl", choices=["jsonl", "h5", "bin"])
+    args = parser.parse_args()
+
+    kept, removed = dedup_jsonl(
+        input_dir=args.input_dir,
+        output_dir=args.output_dir,
+        threshold=args.threshold,
+        num_perm=args.num_perm,
+        ngram=args.ngram,
+        output_format=args.output_format,
+    )
+
+    total = kept + removed
+    print(f"kept={kept}, removed={removed} ({removed/max(total,1)*100:.1f}%)")
+
+
+if __name__ == "__main__":
+    main()