ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: 重命名pretrain 和 sft 脚本

Browse Source
This commit is contained in:
ViperEkura
2025-07-21 19:58:15 +08:00
parent c2821c150a
commit f9ce67e03c
3 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pre_train_file.py → dump_pt_file.py
View File
sft_file.py → dump_sft_file.py
View File
utils.py
+1 -1
View File
@@ -97,8 +97,8 @@ def dump_pkl_files(
output_package: Dict[str, Tensor] = {} output_package: Dict[str, Tensor] = {}
for key in output_keys: for key in output_keys:
print(f"Packaging key: '{key}'")
if packing_size > 0: if packing_size > 0:
print(f"Packaging key: '{key}'")
arrows[key] = pack_sequences(arrows[key], packing_size, pad_value) arrows[key] = pack_sequences(arrows[key], packing_size, pad_value)
sequence = torch.cat(arrows[key]) sequence = torch.cat(arrows[key])
output_package[key] = sequence output_package[key] = sequence