refactor(dataset processing): 重构数据集处理流程并添加新功能

english-fineweb.py

@@ -1,16 +1,4 @@
 from utils import process_dataset
-import re
-
-def comprehensive_normalization(text):
-    replacements = {
-        '\u2018': "'", '\u2019': "'", '\u0060': "'",
-        '\u201C': '"', '\u201D': '"', 
-        '\u2013': '-', '\u2014': '--', '\u2212': '-',
-        '\u00A0': ' ',
-        '\u2026': '...'
-    }
-    pattern = re.compile('|'.join(re.escape(k) for k in replacements))
-    return pattern.sub(lambda m: replacements[m.group()], text)
 
 
 if __name__ == "__main__":
@@ -18,5 +6,4 @@ if __name__ == "__main__":
         dataset_name="HuggingFaceFW/fineweb",
         output_subdir="english-fineweb",
         dataset_config="sample-10BT",
-        normalization_func=comprehensive_normalization
     )

english-wiki.py

@@ -0,0 +1,7 @@
+from utils import process_dataset
+
+if __name__ == "__main__":
+    process_dataset(
+        dataset_name="Blaze7451/enwiki_structured_content",
+        output_subdir="english-wiki"
+    )

utils.py

@@ -1,6 +1,19 @@
 from datasets import load_dataset
 import json
 import os
+import re
+
+
+def comprehensive_normalization(text):
+    replacements = {
+        '\u2018': "'", '\u2019': "'", '\u0060': "'",
+        '\u201C': '"', '\u201D': '"', 
+        '\u2013': '-', '\u2014': '--', '\u2212': '-',
+        '\u00A0': ' ',
+        '\u2026': '...'
+    }
+    pattern = re.compile('|'.join(re.escape(k) for k in replacements))
+    return pattern.sub(lambda m: replacements[m.group()], text)      
 
 def process_dataset(
     dataset_name: str,
@@ -8,7 +21,7 @@ def process_dataset(
     dataset_config: str = None,
     split_name: str = "train",
     chunk_size: int = 1000000,
-    normalization_func=None
+    normalization_func=comprehensive_normalization
 ):
 
     dataset_dict = load_dataset(dataset_name, dataset_config) if dataset_config else load_dataset(dataset_name)
@@ -37,3 +50,4 @@ def process_dataset(
 
         print(f"Saved text chunk {i} to {output_path}")
         
+