From d8643d11ddd30ef5517fd3d2f153688851d44104 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Mon, 14 Jul 2025 10:57:23 +0800
Subject: [PATCH] =?UTF-8?q?refactor(dataset=20processing):=20=E9=87=8D?=
 =?UTF-8?q?=E6=9E=84=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=84=E7=90=86=E6=B5=81?=
 =?UTF-8?q?=E7=A8=8B=E5=B9=B6=E6=B7=BB=E5=8A=A0=E6=96=B0=E5=8A=9F=E8=83=BD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 english-fineweb.py | 13 -------------
 english-wiki.py    |  7 +++++++
 utils.py           | 16 +++++++++++++++-
 3 files changed, 22 insertions(+), 14 deletions(-)
 create mode 100644 english-wiki.py

diff --git a/english-fineweb.py b/english-fineweb.py
index f429e6b..3333b4f 100644
--- a/english-fineweb.py
+++ b/english-fineweb.py
@@ -1,16 +1,4 @@
 from utils import process_dataset
-import re
-
-def comprehensive_normalization(text):
-    replacements = {
-        '\u2018': "'", '\u2019': "'", '\u0060': "'",
-        '\u201C': '"', '\u201D': '"', 
-        '\u2013': '-', '\u2014': '--', '\u2212': '-',
-        '\u00A0': ' ',
-        '\u2026': '...'
-    }
-    pattern = re.compile('|'.join(re.escape(k) for k in replacements))
-    return pattern.sub(lambda m: replacements[m.group()], text)
 
 
 if __name__ == "__main__":
@@ -18,5 +6,4 @@ if __name__ == "__main__":
         dataset_name="HuggingFaceFW/fineweb",
         output_subdir="english-fineweb",
         dataset_config="sample-10BT",
-        normalization_func=comprehensive_normalization
     )
\ No newline at end of file
diff --git a/english-wiki.py b/english-wiki.py
new file mode 100644
index 0000000..d9f7ae1
--- /dev/null
+++ b/english-wiki.py
@@ -0,0 +1,7 @@
+from utils import process_dataset
+
+if __name__ == "__main__":
+    process_dataset(
+        dataset_name="Blaze7451/enwiki_structured_content",
+        output_subdir="english-wiki"
+    )
\ No newline at end of file
diff --git a/utils.py b/utils.py
index e3b3ddd..1e15b6a 100644
--- a/utils.py
+++ b/utils.py
@@ -1,6 +1,19 @@
 from datasets import load_dataset
 import json
 import os
+import re
+
+
+def comprehensive_normalization(text):
+    replacements = {
+        '\u2018': "'", '\u2019': "'", '\u0060': "'",
+        '\u201C': '"', '\u201D': '"', 
+        '\u2013': '-', '\u2014': '--', '\u2212': '-',
+        '\u00A0': ' ',
+        '\u2026': '...'
+    }
+    pattern = re.compile('|'.join(re.escape(k) for k in replacements))
+    return pattern.sub(lambda m: replacements[m.group()], text)      
 
 def process_dataset(
     dataset_name: str,
@@ -8,7 +21,7 @@ def process_dataset(
     dataset_config: str = None,
     split_name: str = "train",
     chunk_size: int = 1000000,
-    normalization_func=None
+    normalization_func=comprehensive_normalization
 ):
 
     dataset_dict = load_dataset(dataset_name, dataset_config) if dataset_config else load_dataset(dataset_name)
@@ -37,3 +50,4 @@ def process_dataset(
 
         print(f"Saved text chunk {i} to {output_path}")
         
+