From d4df706bc5fe271e509dd3a6ad455d2e468826e2 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Fri, 11 Jul 2025 10:55:28 +0800
Subject: [PATCH] =?UTF-8?q?fix(dataset):=20=E4=BF=AE=E6=AD=A3=E6=95=B0?=
 =?UTF-8?q?=E6=8D=AE=E9=9B=86=E8=BE=93=E5=87=BA=E8=B7=AF=E5=BE=84=E5=92=8C?=
 =?UTF-8?q?=E5=90=8D=E7=A7=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 chinese-c4.py      | 2 +-
 chinese-wiki.py    | 4 ++--
 english-fineweb.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/chinese-c4.py b/chinese-c4.py
index f0934f6..172270c 100644
--- a/chinese-c4.py
+++ b/chinese-c4.py
@@ -18,7 +18,7 @@ if __name__ == "__main__":
         end_idx = min((i + 1) * chunk_size, total_samples)
         chunk = train_dataset.select(range(start_idx, end_idx))
 
-        output_path = f"{output_dir}/{output_dir}_text_chunk_{i}.jsonl"
+        output_path = f"{output_dir}/chinese-c4_text_chunk_{i}.jsonl"
         with open(output_path, "w", encoding="utf-8") as f:
             for example in chunk:
                 # 每行写入一个 {"text": "xxx"} 对象
diff --git a/chinese-wiki.py b/chinese-wiki.py
index 2a2e13a..815a811 100644
--- a/chinese-wiki.py
+++ b/chinese-wiki.py
@@ -10,7 +10,7 @@ if __name__ == "__main__":
     total_samples = len(train_dataset)
     num_chunks = (total_samples // chunk_size) + 1
     script_dir = os.path.dirname(os.path.abspath(__file__))
-    output_dir = os.path(script_dir, "dataset", "chinese-c4")
+    output_dir = os.path.join(script_dir, "dataset", "chinese-wiki")
     os.makedirs(output_dir, exist_ok=True)
     
     
@@ -19,7 +19,7 @@ if __name__ == "__main__":
         end_idx = min((i + 1) * chunk_size, total_samples)
         chunk = train_dataset.select(range(start_idx, end_idx))
 
-        output_path = f"{output_dir}/{output_dir}_text_chunk_{i}.jsonl"
+        output_path = f"{output_dir}/chinese-wiki_text_chunk_{i}.jsonl"
         with open(output_path, "w", encoding="utf-8") as f:
             for example in chunk:
                 json_line = {"text": example["text"]}
diff --git a/english-fineweb.py b/english-fineweb.py
index ecf929c..871d606 100644
--- a/english-fineweb.py
+++ b/english-fineweb.py
@@ -33,7 +33,7 @@ if __name__ == "__main__":
         end_idx = min((i + 1) * chunk_size, total_samples)
         chunk = train_dataset.select(range(start_idx, end_idx))
         
-        output_path = f"{output_dir}/{output_dir}text_chunk_{i}.jsonl"
+        output_path = f"{output_dir}/english-fineweb_text_chunk_{i}.jsonl"
         with open(output_path, "w", encoding="utf-8") as f:
             for example in chunk:
                 json_line = {"text": comprehensive_normalization(example["text"])}