From f7200ee4cf21857192246db3b48e9b09a0014ace Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Wed, 6 Aug 2025 09:58:15 +0800
Subject: [PATCH] =?UTF-8?q?feat(pre=5Ftrain):=20=E6=9B=B4=E6=96=B0=20Chine?=
 =?UTF-8?q?se=20Cosmopedia=20=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=84=E7=90=86?=
 =?UTF-8?q?=E8=84=9A=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pre_train/chinese-cosmopedia.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/pre_train/chinese-cosmopedia.py b/pre_train/chinese-cosmopedia.py
index b6cd249..b0d1805 100644
--- a/pre_train/chinese-cosmopedia.py
+++ b/pre_train/chinese-cosmopedia.py
@@ -2,17 +2,15 @@ from datasets import load_dataset
 from modules.utils import process_dataset
 
 if __name__ == "__main__":
-    max_chunk_num = 10
     chunk_size = 1000000
     
     dataset = load_dataset(
         "opencsg/chinese-cosmopedia", 
-        data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]}
+        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]}
     )
     
     process_dataset(
         dataset_dict=dataset,
         output_subdir="chinese-wiki-pretrain",
-        max_chunk_num=max_chunk_num,
         chunk_size=chunk_size,
     )
\ No newline at end of file