ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(pre_train): 更新 Chinese Cosmopedia 数据集处理脚本

Browse Source
This commit is contained in:
ViperEkura
2025-08-06 09:58:15 +08:00
parent d9ef711e28
commit f7200ee4cf
1 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pre_train/chinese-cosmopedia.py
+1 -3
View File
@@ -2,17 +2,15 @@ from datasets import load_dataset
from modules.utils import process_dataset from modules.utils import process_dataset
if __name__ == "__main__": if __name__ == "__main__":
max_chunk_num = 10
chunk_size = 1000000 chunk_size = 1000000
dataset = load_dataset( dataset = load_dataset(
"opencsg/chinese-cosmopedia", "opencsg/chinese-cosmopedia",
data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]} data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]}
) )
process_dataset( process_dataset(
dataset_dict=dataset, dataset_dict=dataset,
output_subdir="chinese-wiki-pretrain", output_subdir="chinese-wiki-pretrain",
max_chunk_num=max_chunk_num,
chunk_size=chunk_size, chunk_size=chunk_size,
) )