From fec37545e2d211f7eb2293bb40677cade2de6957 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Mon, 6 Jul 2026 08:53:18 +0800
Subject: [PATCH] Add max_chunks=18 to dolma-30b export, remove max_chunks=5
 from english-wiki, rename project to datapipline

---
 pyproject.toml                     |  2 +-
 scripts/pre_train/english-dolma.py | 11 +++++++++++
 scripts/pre_train/english-wiki.py  |  1 -
 3 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 scripts/pre_train/english-dolma.py

diff --git a/pyproject.toml b/pyproject.toml
index fdfa49a..e7a159e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,7 +3,7 @@ requires = ["setuptools>=64", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "khaosz_dataset"
+name = "datapipline"
 version = "0.1.0"
 description = "A dataset processing toolkit for language model training"
 readme = "README.md"
diff --git a/scripts/pre_train/english-dolma.py b/scripts/pre_train/english-dolma.py
new file mode 100644
index 0000000..07ef4c7
--- /dev/null
+++ b/scripts/pre_train/english-dolma.py
@@ -0,0 +1,11 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("emozilla/dolma-v1_7-30B")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="english-dolma-30b-pretrain",
+        max_chunks=18,
+    )
diff --git a/scripts/pre_train/english-wiki.py b/scripts/pre_train/english-wiki.py
index fd54ed4..fe4cca2 100644
--- a/scripts/pre_train/english-wiki.py
+++ b/scripts/pre_train/english-wiki.py
@@ -7,5 +7,4 @@ if __name__ == "__main__":
         dataset=dataset["train"],
         output_dir="./dataset",
         output_prefix="english-wiki-pretrain",
-        max_chunks=5,
     )