diff --git a/english-fineweb.py b/english-fineweb.py index f429e6b..3333b4f 100644 --- a/english-fineweb.py +++ b/english-fineweb.py @@ -1,16 +1,4 @@ from utils import process_dataset -import re - -def comprehensive_normalization(text): - replacements = { - '\u2018': "'", '\u2019': "'", '\u0060': "'", - '\u201C': '"', '\u201D': '"', - '\u2013': '-', '\u2014': '--', '\u2212': '-', - '\u00A0': ' ', - '\u2026': '...' - } - pattern = re.compile('|'.join(re.escape(k) for k in replacements)) - return pattern.sub(lambda m: replacements[m.group()], text) if __name__ == "__main__": @@ -18,5 +6,4 @@ if __name__ == "__main__": dataset_name="HuggingFaceFW/fineweb", output_subdir="english-fineweb", dataset_config="sample-10BT", - normalization_func=comprehensive_normalization ) \ No newline at end of file diff --git a/english-wiki.py b/english-wiki.py new file mode 100644 index 0000000..d9f7ae1 --- /dev/null +++ b/english-wiki.py @@ -0,0 +1,7 @@ +from utils import process_dataset + +if __name__ == "__main__": + process_dataset( + dataset_name="Blaze7451/enwiki_structured_content", + output_subdir="english-wiki" + ) \ No newline at end of file diff --git a/utils.py b/utils.py index e3b3ddd..1e15b6a 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,19 @@ from datasets import load_dataset import json import os +import re + + +def comprehensive_normalization(text): + replacements = { + '\u2018': "'", '\u2019': "'", '\u0060': "'", + '\u201C': '"', '\u201D': '"', + '\u2013': '-', '\u2014': '--', '\u2212': '-', + '\u00A0': ' ', + '\u2026': '...' + } + pattern = re.compile('|'.join(re.escape(k) for k in replacements)) + return pattern.sub(lambda m: replacements[m.group()], text) def process_dataset( dataset_name: str, @@ -8,7 +21,7 @@ def process_dataset( dataset_config: str = None, split_name: str = "train", chunk_size: int = 1000000, - normalization_func=None + normalization_func=comprehensive_normalization ): dataset_dict = load_dataset(dataset_name, dataset_config) if dataset_config else load_dataset(dataset_name) @@ -37,3 +50,4 @@ def process_dataset( print(f"Saved text chunk {i} to {output_path}") +