ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(utils): 更新缓存文件键名

Browse Source
This commit is contained in:
ViperEkura
2025-08-11 13:49:58 +08:00
parent 96f3eb27fb
commit be79108f84
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/utils.py
+2 -2
View File
@@ -176,10 +176,10 @@ def cache_files(tokenizer, files, base_out_dir, cache_type):
keys = [] keys = []
if cache_type == "pt": if cache_type == "pt":
processor = get_pt_processor(tokenizer) processor = get_pt_processor(tokenizer)
keys = ["text"] keys = ["sequence"]
elif cache_type == "sft": elif cache_type == "sft":
processor = get_sft_processor(tokenizer) processor = get_sft_processor(tokenizer)
keys = ["query", "response"] keys = ["sequence", "mask"]
elif cache_type == "dpo": elif cache_type == "dpo":
processor = get_dpo_processor(tokenizer) processor = get_dpo_processor(tokenizer)
keys = ["chosen", "chosen_mask", "rejected", "rejected_mask"] keys = ["chosen", "chosen_mask", "rejected", "rejected_mask"]