From 6edc31ce2ec0cce2ea5a90ed9068e91887c79850 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Tue, 15 Jul 2025 14:06:18 +0800
Subject: [PATCH] =?UTF-8?q?refactor(dataset):=20=E6=9B=B4=E6=96=B0?=
 =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E5=8A=A0=E8=BD=BD=E8=B7=AF=E5=BE=84?=
 =?UTF-8?q?=E5=92=8C=E5=A4=84=E7=90=86=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 chinese-cosmopedia.py | 2 +-
 utils.py              | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/chinese-cosmopedia.py b/chinese-cosmopedia.py
index f451dc2..94d7ba0 100644
--- a/chinese-cosmopedia.py
+++ b/chinese-cosmopedia.py
@@ -7,7 +7,7 @@ if __name__ == "__main__":
     
     dataset = load_dataset(
         "opencsg/chinese-cosmopedia", 
-        data_files=[f"0000{i}.parquet" for i in range(5)]
+        data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]}
     )
     
     process_dataset(
diff --git a/utils.py b/utils.py
index 83ea164..d62d4ac 100644
--- a/utils.py
+++ b/utils.py
@@ -66,7 +66,6 @@ def process_dataset(
     column_name: str = "text",
     normalization_func=comprehensive_normalization,
 ):
-
     train_dataset = dataset_dict[split_name]
     total_samples = len(train_dataset)
     num_chunks = (total_samples // chunk_size) + 1