From 9a417ab6a5006b3ee870f03aa4cbcf69a711c2e5 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Fri, 11 Jul 2025 10:31:10 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E4=B8=AD=E6=96=87?=
 =?UTF-8?q?=E7=BB=B4=E5=9F=BA=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=84=E7=90=86?=
 =?UTF-8?q?=E8=84=9A=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitignore      |  3 +++
 chinese-wiki.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 chinese-wiki.py

diff --git a/.gitignore b/.gitignore
index 4204de1..2d06cf2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,6 @@ pkl_output/*
 
 # tokenzier
 tokenizer.json
+
+# test unit
+test*
diff --git a/chinese-wiki.py b/chinese-wiki.py
new file mode 100644
index 0000000..2a2e13a
--- /dev/null
+++ b/chinese-wiki.py
@@ -0,0 +1,28 @@
+from datasets import load_dataset
+import json
+import os
+
+
+if __name__ == "__main__":
+    dataset_dict = load_dataset("Blaze7451/Wiki-zh-20250601")
+    train_dataset = dataset_dict["train"]
+    chunk_size = 1000000
+    total_samples = len(train_dataset)
+    num_chunks = (total_samples // chunk_size) + 1
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    output_dir = os.path(script_dir, "dataset", "chinese-c4")
+    os.makedirs(output_dir, exist_ok=True)
+    
+    
+    for i in range(num_chunks):
+        start_idx = i * chunk_size
+        end_idx = min((i + 1) * chunk_size, total_samples)
+        chunk = train_dataset.select(range(start_idx, end_idx))
+
+        output_path = f"{output_dir}/{output_dir}_text_chunk_{i}.jsonl"
+        with open(output_path, "w", encoding="utf-8") as f:
+            for example in chunk:
+                json_line = {"text": example["text"]}
+                f.write(json.dumps(json_line, ensure_ascii=False) + "\n")
+
+        print(f"Saved text chunk {i} to {output_path}")
\ No newline at end of file