refactor(chinese-wiki): 更新数据集处理脚本

2025-07-14 13:28:17 +08:00
parent d8643d11dd
commit 2b0f805ff3
2 changed files with 4 additions and 3 deletions
@@ -24,7 +24,7 @@ def process_dataset(
    normalization_func=comprehensive_normalization
 ):

-    dataset_dict = load_dataset(dataset_name, dataset_config) if dataset_config else load_dataset(dataset_name)
+    dataset_dict = load_dataset(dataset_name, dataset_config)
    train_dataset = dataset_dict[split_name]

    total_samples = len(train_dataset)