ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修复未创建文件夹的bug

Browse Source
This commit is contained in:
ViperEkura
2025-06-30 21:45:51 +08:00
parent 0690f66bed
commit 6238764047
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
to_ids.py
+4
View File
@@ -25,6 +25,10 @@ def process_files(tokenizer: BpeTokenizer, files: List[str], base_out_dir):
for file_path in tqdm(files, desc="Processing files", total=len(files)): for file_path in tqdm(files, desc="Processing files", total=len(files)):
out_file_name = os.path.basename(file_path).replace(".jsonl", ".pkl") out_file_name = os.path.basename(file_path).replace(".jsonl", ".pkl")
out_file_path = os.path.join(base_out_dir, out_file_name) out_file_path = os.path.join(base_out_dir, out_file_name)
if not os.path.exists(out_file_path):
os.makedirs(os.path.dirname(out_file_path), exist_ok=True)
convert_to_ids(tokenizer, file_path, out_file_path) convert_to_ids(tokenizer, file_path, out_file_path)