ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(chinese-wiki): 更新数据集处理脚本

Browse Source
This commit is contained in:
ViperEkura
2025-07-14 13:28:17 +08:00
parent d8643d11dd
commit 2b0f805ff3
2 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chinese-wiki.py
+3 -2
View File
@@ -2,6 +2,7 @@ from utils import process_dataset
if __name__ == "__main__": if __name__ == "__main__":
process_dataset( process_dataset(
dataset_name="Blaze7451/Wiki-zh-20250601", dataset_name="wikimedia/wikipedia",
output_subdir="chinese-wiki" output_subdir="chinese-wiki",
dataset_config="zh"
) )
utils.py
+1 -1
View File
@@ -24,7 +24,7 @@ def process_dataset(
normalization_func=comprehensive_normalization normalization_func=comprehensive_normalization
): ):
dataset_dict = load_dataset(dataset_name, dataset_config) if dataset_config else load_dataset(dataset_name) dataset_dict = load_dataset(dataset_name, dataset_config)
train_dataset = dataset_dict[split_name] train_dataset = dataset_dict[split_name]
total_samples = len(train_dataset) total_samples = len(train_dataset)