ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(dataset): 更新数据集加载路径和处理代码

Browse Source
This commit is contained in:
ViperEkura
2025-07-15 14:06:18 +08:00
parent e77fb56c45
commit 6edc31ce2e
2 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chinese-cosmopedia.py
+1 -1
View File
@@ -7,7 +7,7 @@ if __name__ == "__main__":
dataset = load_dataset( dataset = load_dataset(
"opencsg/chinese-cosmopedia", "opencsg/chinese-cosmopedia",
data_files=[f"0000{i}.parquet" for i in range(5)] data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]}
) )
process_dataset( process_dataset(
utils.py
-1
View File
@@ -66,7 +66,6 @@ def process_dataset(
column_name: str = "text", column_name: str = "text",
normalization_func=comprehensive_normalization, normalization_func=comprehensive_normalization,
): ):
train_dataset = dataset_dict[split_name] train_dataset = dataset_dict[split_name]
total_samples = len(train_dataset) total_samples = len(train_dataset)
num_chunks = (total_samples // chunk_size) + 1 num_chunks = (total_samples // chunk_size) + 1