refactor: 修改项目结构

scripts/pre_train/chinese-c4.py

@@ -0,0 +1,10 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("shjwudp/chinese-c4")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="chinese-c4-pretrain",
+    )

scripts/pre_train/chinese-cosmopedia.py

@@ -0,0 +1,14 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset(
+        "opencsg/chinese-cosmopedia",
+        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]}
+    )
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="chinese-cosmopedia-pretrain",
+        chunk_size=1_000_000,
+    )

scripts/pre_train/english-fineweb.py

@@ -0,0 +1,10 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="english-fineweb-pretrain",
+    )

scripts/pre_train/english-wiki.py

@@ -0,0 +1,11 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("Blaze7451/enwiki_structured_content")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="english-wiki-pretrain",
+        max_chunks=5,
+    )