refactor(chinese-wiki): 更新数据集处理脚本

2025-07-14 13:28:17 +08:00
parent d8643d11dd
commit 2b0f805ff3
2 changed files with 4 additions and 3 deletions
@@ -2,6 +2,7 @@ from utils import process_dataset

 if __name__ == "__main__":
    process_dataset(
-        dataset_name="Blaze7451/Wiki-zh-20250601",
-        output_subdir="chinese-wiki"
+        dataset_name="wikimedia/wikipedia",
+        output_subdir="chinese-wiki",
+        dataset_config="zh"
    )
@@ -24,7 +24,7 @@ def process_dataset(
    normalization_func=comprehensive_normalization
 ):

-    dataset_dict = load_dataset(dataset_name, dataset_config) if dataset_config else load_dataset(dataset_name)
+    dataset_dict = load_dataset(dataset_name, dataset_config)
    train_dataset = dataset_dict[split_name]

    total_samples = len(train_dataset)