feat: add MetaMathQA SFT export script and fix ChatML newline encoding

2026-07-02 17:40:03 +08:00
parent fd99b9d852
commit 2f919e9243
2 changed files with 26 additions and 8 deletions
@@ -0,0 +1,17 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(sample: dict) -> dict:
+    return {"query": sample["query"], "response": sample["response"]}
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("meta-math/MetaMathQA", split="train")
+    export_dataset(
+        dataset=dataset,
+        output_dir="./dataset",
+        output_prefix="MetaMathQA",
+        process_func=process_func,
+        chunk_size=1_000_000,
+    )