refactor: 重构打包模块，新增 BFD/FFD/Greedy 三种 bin-packing 算法，默认 BFD

- 将 pipeline/packing.py 拆分为 packing/ 子包 (base/stream/binpack)
- 新增 BfdPacker(默认)/FfDPacker/GreedyPacker，移除 StreamingPacker
- 超长序列直接截断至 pack_size
- group_size 语义改为"每 N 个 chunk 合并为一块"，默认 1000
- 新增 AutoTokenizer.token_to_id()，修复 ChatML 中 hacky 的 nl_id 获取
- pad_value 默认改为 2(pad_token_id)，position_ids pad=0, loss_mask pad=False
- 新增 position_ids 打包后归零一致性测试
- scripts/cache_h5.py 新增 --pack-algo 参数

scripts/cache_h5.py

@@ -43,6 +43,13 @@ def main():
         default=None,
         help="Prompt strategy: chatml, alpaca (default: chatml)",
     )
+    parser.add_argument(
+        "-a",
+        "--pack-algo",
+        default=None,
+        choices=[None, "bfd", "ffd", "greedy"],
+        help="Packing algorithm: bfd (default), ffd, greedy",
+    )
     parser.add_argument(
         "-p",
         "--pack-size",
@@ -51,7 +58,14 @@ def main():
         help="Pack size, <=0 to disable (default: -1)",
     )
     parser.add_argument(
-        "--pad-value", type=int, default=0, help="Padding value (default: 0)"
+        "--pad-value", type=int, default=2, help="Padding token ID (default: 2 = <|pad|>)"
+    )
+    parser.add_argument(
+        "-g",
+        "--group-size",
+        type=int,
+        default=1_000,
+        help="Merge every N packed chunks into one tensor, <=0 to disable (default: 1000)",
     )
     parser.add_argument(
         "--log-level",
@@ -95,9 +109,14 @@ def main():
 
     print(f"\nStart caching...")
     if args.pack_size > 0:
-        print(f"  pack_size={args.pack_size}, pad_value={args.pad_value}")
+        algo = args.pack_algo or "bfd"
+        print(f"  pack_size={args.pack_size}, pad_value={args.pad_value}, algo={algo}")
     else:
         print(f"  no packing")
+    if args.group_size > 0:
+        print(f"  group_size={args.group_size} chunks per tensor")
+    else:
+        print(f"  no grouping")
 
     cache_jsonl(
         files=jsonl_files,
@@ -105,6 +124,8 @@ def main():
         processor=processor,
         pack_size=args.pack_size,
         pad_value=args.pad_value,
+        group_size=args.group_size,
+        pack_algo=args.pack_algo,
     )
     print(f"\nDone! Output saved to {output_dir}")