ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(chinese-cosmopedia): 更新数据集加载方式并调整相关参数

Browse Source
This commit is contained in:
ViperEkura
2025-07-15 12:54:31 +08:00
parent bea4f83842
commit e6f5b26a75
2 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chinese-cosmopedia.py
+2 -2
View File
@@ -6,9 +6,9 @@ if __name__ == "__main__":
item_size = max_chunk_num * chunk_size item_size = max_chunk_num * chunk_size
process_dataset( process_dataset(
dataset_name="chinese-cosmopedia", dataset_name="opencsg/chinese-cosmopedia",
output_subdir="chinese-wiki", output_subdir="chinese-wiki",
split=f"train[:{item_size}]", data_files=[f"0000{i}.parquet" for i in range(5)],
max_chunk_num=max_chunk_num, max_chunk_num=max_chunk_num,
chunk_size=chunk_size, chunk_size=chunk_size,
) )
utils.py
+5 -3
View File
@@ -63,6 +63,7 @@ def process_dataset(
dataset_config: str = None, dataset_config: str = None,
max_chunk_num: int = None, max_chunk_num: int = None,
split: str = None, split: str = None,
data_files: List[str] = None,
split_name: str = "train", split_name: str = "train",
column_name: str = "text", column_name: str = "text",
chunk_size: int = 1000000, chunk_size: int = 1000000,
@@ -70,9 +71,10 @@ def process_dataset(
): ):
dataset_dict = load_dataset( dataset_dict = load_dataset(
data_dir=dataset_name, path=dataset_name,
data_files=dataset_config, name=dataset_config,
split=split split=split,
data_files=data_files
) )
train_dataset = dataset_dict[split_name] train_dataset = dataset_dict[split_name]