ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(to_ids): 注释掉中文维基百科数据集的处理

Browse Source
This commit is contained in:
ViperEkura
2025-07-15 07:01:26 +08:00
parent 95a3568328
commit 1c7e49a631
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
to_ids.py
+1 -1
View File
@@ -11,7 +11,7 @@ if __name__ == "__main__":
# os.path.join("dataset", "chinese-c4"), # os.path.join("dataset", "chinese-c4"),
# os.path.join("dataset", "english-fineweb"), # os.path.join("dataset", "english-fineweb"),
# os.path.join("dataset", "english-wiki"), # os.path.join("dataset", "english-wiki"),
os.path.join("dataset", "chinese-wiki"), # os.path.join("dataset", "chinese-wiki"),
] ]
base_out_dir = "pkl_output" base_out_dir = "pkl_output"
files = [] files = []