From c4ce882d56d71aa875bde00f4801bcdc79103111 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Fri, 1 Aug 2025 17:28:23 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20Magpie-Pro-300K=20?=
 =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E7=9A=84=20SFT=20=E5=A4=84=E7=90=86?=
 =?UTF-8?q?=E8=84=9A=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 sft_magpie-pro-300k.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 sft_magpie-pro-300k.py

diff --git a/sft_magpie-pro-300k.py b/sft_magpie-pro-300k.py
new file mode 100644
index 0000000..897852c
--- /dev/null
+++ b/sft_magpie-pro-300k.py
@@ -0,0 +1,30 @@
+# HuggingFaceTB/Magpie-Pro-300K-Filtered-H4
+from datasets import load_dataset
+from utils import process_dataset
+
+
+def process_func(input_dict: dict):
+    conversations = input_dict["conversations"]
+    assert len(conversations) % 2 == 0
+    n = len(conversations) // 2
+    examples = []
+    
+    for i in range(n):
+        user_msg = conversations[2*i]["value"]
+        assistant_msg = conversations[2*i+1]["value"]
+        examples.append({
+            "query": user_msg,
+            "response": assistant_msg
+        })
+    
+    return examples
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("HuggingFaceTB/Magpie-Pro-300K-Filtered-H4")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="belle-sft",
+        process_func=process_func,
+        split_name="train-sft"
+    )
\ No newline at end of file