ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修复输出文件的bug

Browse Source
This commit is contained in:
ViperEkura
2025-06-30 22:37:49 +08:00
parent b508402724
commit a8f05cccd6
2 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -3,6 +3,7 @@ __pycache__*
# dataset # dataset
dataset/* dataset/*
pkl_output/*
# tokenzier # tokenzier
tokenizer.json tokenizer.json
to_ids.py
+2 -2
View File
@@ -20,7 +20,7 @@ def convert_to_ids(tokenizer: BpeTokenizer, file_path, out_file_path):
arrow = torch.tensor(ids, dtype=torch.int32) arrow = torch.tensor(ids, dtype=torch.int32)
arrows.append(arrow) arrows.append(arrow)
with open(out_file_path, "w") as f: with open(out_file_path, "wb") as f:
tensor = torch.cat(arrows) tensor = torch.cat(arrows)
pkl.dump(tensor, f) pkl.dump(tensor, f)
@@ -41,7 +41,7 @@ if __name__ == "__main__":
os.path.join("dataset", "chinese-c4"), os.path.join("dataset", "chinese-c4"),
os.path.join("dataset", "english-fineweb") os.path.join("dataset", "english-fineweb")
] ]
base_out_dir = "cache" base_out_dir = "pkl_output"
files = [] files = []
for dir_path in base_dir: for dir_path in base_dir: