ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(utils): 为缓存文件函数添加 packing 和 padding 参数支持

Browse Source
This commit is contained in:
ViperEkura
2025-10-02 22:01:00 +08:00
parent 2ccb26add4
commit cd3a22c1ac
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/utils.py
+2 -2
View File
@@ -162,7 +162,7 @@ def get_dpo_processor(tokenizer: BpeTokenizer):
return processor return processor
def cache_files(tokenizer, files, base_out_dir, cache_type): def cache_files(tokenizer, files, base_out_dir, cache_type, packing_size: int = -1, pad_value: int = 1):
processor = None processor = None
keys = [] keys = []
if cache_type == "pt": if cache_type == "pt":
@@ -177,7 +177,7 @@ def cache_files(tokenizer, files, base_out_dir, cache_type):
else: else:
raise ValueError("Invalid cache type") raise ValueError("Invalid cache type")
dump_pkl_files(files, base_out_dir, processor, keys) dump_pkl_files(files, base_out_dir, processor, keys, packing_size, pad_value)
def process_dataset( def process_dataset(