refactor: 重构打包模块,新增 BFD/FFD/Greedy 三种 bin-packing 算法,默认 BFD

- 将 pipeline/packing.py 拆分为 packing/ 子包 (base/stream/binpack)
- 新增 BfdPacker(默认)/FfDPacker/GreedyPacker,移除 StreamingPacker
- 超长序列直接截断至 pack_size
- group_size 语义改为"每 N 个 chunk 合并为一块",默认 1000
- 新增 AutoTokenizer.token_to_id(),修复 ChatML 中 hacky 的 nl_id 获取
- pad_value 默认改为 2(pad_token_id),position_ids pad=0, loss_mask pad=False
- 新增 position_ids 打包后归零一致性测试
- scripts/cache_h5.py 新增 --pack-algo 参数
This commit is contained in:
2026-07-03 16:17:27 +08:00
parent 2f919e9243
commit 598e1ce4ae
14 changed files with 715 additions and 295 deletions
+4 -1
View File
@@ -145,6 +145,9 @@ class DummyTokenizer:
def encode(self, text: str, add_special_tokens: bool = False):
return [ord(c) for c in text]
def token_to_id(self, token: str):
return ord(token)
def apply_chat_template(
self, messages, add_generation_prompt=True, tokenize=True
):
@@ -168,7 +171,7 @@ class TestPositionIds:
processor = SFTProcessor(DummyTokenizer())
out_dir = os.path.join(tmpdir, "cached")
cache_jsonl([jsonl_path], out_dir, processor, pack_size=-1)
cache_jsonl([jsonl_path], out_dir, processor, pack_size=-1, group_size=0)
h5_path = os.path.join(out_dir, "data.h5")
loaded = HDF5Handler.load(h5_path, share_memory=False)