From e9996290989ae6f4a3453323ea691ad7cc69e34a Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Sun, 26 Jul 2026 10:57:30 +0800
Subject: [PATCH] =?UTF-8?q?=E7=A7=BB=E9=99=A4=E7=BC=93=E5=AD=98=E8=84=9A?=
 =?UTF-8?q?=E6=9C=AC=E4=B8=AD=E7=9A=84=20data=5Ffiles=20=E5=92=8C=20max=5F?=
 =?UTF-8?q?chunks=20=E9=99=90=E5=88=B6=E5=8F=82=E6=95=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/pre_train/chinese-cosmopedia.py | 1 -
 scripts/pre_train/english-dolma.py      | 1 -
 2 files changed, 2 deletions(-)

diff --git a/scripts/pre_train/chinese-cosmopedia.py b/scripts/pre_train/chinese-cosmopedia.py
index 8a1e769..01f64b3 100644
--- a/scripts/pre_train/chinese-cosmopedia.py
+++ b/scripts/pre_train/chinese-cosmopedia.py
@@ -4,7 +4,6 @@ from pipeline import export_dataset
 if __name__ == "__main__":
     dataset = load_dataset(
         "opencsg/chinese-cosmopedia",
-        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]},
     )
     export_dataset(
         dataset=dataset["train"],
diff --git a/scripts/pre_train/english-dolma.py b/scripts/pre_train/english-dolma.py
index 07ef4c7..7e82b3c 100644
--- a/scripts/pre_train/english-dolma.py
+++ b/scripts/pre_train/english-dolma.py
@@ -7,5 +7,4 @@ if __name__ == "__main__":
         dataset=dataset["train"],
         output_dir="./dataset",
         output_prefix="english-dolma-30b-pretrain",
-        max_chunks=18,
     )