From 1c132826c067be4cdb118cc19cefbf7707dfcbdb Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Mon, 14 Jul 2025 13:35:06 +0800
Subject: [PATCH] =?UTF-8?q?feat(chinese-wiki.,=20english-wiki.):=20?=
 =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20max=5Fchunk=5Fsize=20=E5=8F=82=E6=95=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 chinese-wiki.py | 3 ++-
 english-wiki.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/chinese-wiki.py b/chinese-wiki.py
index dc979a7..dca111f 100644
--- a/chinese-wiki.py
+++ b/chinese-wiki.py
@@ -4,5 +4,6 @@ if __name__ == "__main__":
     process_dataset(
         dataset_name="wikimedia/wikipedia",
         output_subdir="chinese-wiki",
-        dataset_config="zh"
+        dataset_config="zh",
+        max_chunk_size=5,
     )
\ No newline at end of file
diff --git a/english-wiki.py b/english-wiki.py
index d9f7ae1..cdeca88 100644
--- a/english-wiki.py
+++ b/english-wiki.py
@@ -3,5 +3,6 @@ from utils import process_dataset
 if __name__ == "__main__":
     process_dataset(
         dataset_name="Blaze7451/enwiki_structured_content",
-        output_subdir="english-wiki"
+        output_subdir="english-wiki",
+        max_chunk_size=5,
     )
\ No newline at end of file