Add max_chunks=18 to dolma-30b export, remove max_chunks=5 from english-wiki, rename project to datapipline

pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "khaosz_dataset"
+name = "datapipline"
 version = "0.1.0"
 description = "A dataset processing toolkit for language model training"
 readme = "README.md"

scripts/pre_train/english-dolma.py

@@ -0,0 +1,11 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("emozilla/dolma-v1_7-30B")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="english-dolma-30b-pretrain",
+        max_chunks=18,
+    )

scripts/pre_train/english-wiki.py

@@ -7,5 +7,4 @@ if __name__ == "__main__":
         dataset=dataset["train"],
         output_dir="./dataset",
         output_prefix="english-wiki-pretrain",
-        max_chunks=5,
     )