From 2b0f805ff39456b4faab5076b6aa7c8fe9da3bb6 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Mon, 14 Jul 2025 13:28:17 +0800
Subject: [PATCH] =?UTF-8?q?refactor(chinese-wiki):=20=E6=9B=B4=E6=96=B0?=
 =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=84=E7=90=86=E8=84=9A=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 chinese-wiki.py | 5 +++--
 utils.py        | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/chinese-wiki.py b/chinese-wiki.py
index e0afb99..dc979a7 100644
--- a/chinese-wiki.py
+++ b/chinese-wiki.py
@@ -2,6 +2,7 @@ from utils import process_dataset
 
 if __name__ == "__main__":
     process_dataset(
-        dataset_name="Blaze7451/Wiki-zh-20250601",
-        output_subdir="chinese-wiki"
+        dataset_name="wikimedia/wikipedia",
+        output_subdir="chinese-wiki",
+        dataset_config="zh"
     )
\ No newline at end of file
diff --git a/utils.py b/utils.py
index 1e15b6a..610ab00 100644
--- a/utils.py
+++ b/utils.py
@@ -24,7 +24,7 @@ def process_dataset(
     normalization_func=comprehensive_normalization
 ):
 
-    dataset_dict = load_dataset(dataset_name, dataset_config) if dataset_config else load_dataset(dataset_name)
+    dataset_dict = load_dataset(dataset_name, dataset_config)
     train_dataset = dataset_dict[split_name]
 
     total_samples = len(train_dataset)