feat(project_structure): 重构项目目录并添加运行脚本

pre_train/__init__.py

@@ -0,0 +1,12 @@
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import utils
+from tokenizer import BpeTokenizer
+
+__all__ = [
+    "utils",
+    "BpeTokenizer",
+]

pre_train/chinese-c4.py

@@ -0,0 +1,9 @@
+from datasets import load_dataset
+from utils import process_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("shjwudp/chinese-c4")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="chinese-c4"
+    )

pre_train/chinese-cosmopedia.py

@@ -0,0 +1,18 @@
+from datasets import load_dataset
+from utils import process_dataset
+
+if __name__ == "__main__":
+    max_chunk_num = 10
+    chunk_size = 1000000
+    
+    dataset = load_dataset(
+        "opencsg/chinese-cosmopedia", 
+        data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]}
+    )
+    
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="chinese-wiki",
+        max_chunk_num=max_chunk_num,
+        chunk_size=chunk_size,
+    )

pre_train/english-fineweb.py

@@ -0,0 +1,9 @@
+from datasets import load_dataset
+from utils import process_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="english-fineweb",
+    )

pre_train/english-wiki.py

@@ -0,0 +1,10 @@
+from datasets import load_dataset
+from utils import process_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("Blaze7451/enwiki_structured_content")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="english-wiki",
+        max_chunk_size=5,
+    )