From 6a259cab0d33041a70af5f4959b8713e81ae9c89 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Sun, 10 Aug 2025 11:30:08 +0800
Subject: [PATCH] =?UTF-8?q?refactor(supervised=5Ffinetuning):=20=E7=A7=BB?=
 =?UTF-8?q?=E9=99=A4=20smoltalk-chinese=20=E6=95=B0=E6=8D=AE=E9=9B=86?=
 =?UTF-8?q?=E5=A4=84=E7=90=86=E8=84=9A=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../sft_firefly-1.1m-rephrased.py             | 18 +++++++++++
 supervised_finetuning/sft_small_talk.py       | 30 -------------------
 2 files changed, 18 insertions(+), 30 deletions(-)
 create mode 100644 supervised_finetuning/sft_firefly-1.1m-rephrased.py
 delete mode 100644 supervised_finetuning/sft_small_talk.py

diff --git a/supervised_finetuning/sft_firefly-1.1m-rephrased.py b/supervised_finetuning/sft_firefly-1.1m-rephrased.py
new file mode 100644
index 0000000..673850e
--- /dev/null
+++ b/supervised_finetuning/sft_firefly-1.1m-rephrased.py
@@ -0,0 +1,18 @@
+# Mxode/Firefly-1.1M-Rephrased
+from datasets import load_dataset
+from modules.utils import process_dataset
+
+def process_func(input_dict: dict):
+    instruction = input_dict["instruction"]
+    output = input_dict["output"]
+    return {"query": instruction, "response": output}
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("Mxode/Firefly-1.1M-Rephrased")
+    
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="Firefly-1.1M-Rephrased",
+        process_func=process_func
+    )
diff --git a/supervised_finetuning/sft_small_talk.py b/supervised_finetuning/sft_small_talk.py
deleted file mode 100644
index a0e9028..0000000
--- a/supervised_finetuning/sft_small_talk.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# opencsg/smoltalk-chinese
-from datasets import load_dataset
-from modules.utils import process_dataset
-
-
-def process_func(input_dict: dict):
-    conversations = input_dict["conversations"]
-    assert len(conversations) % 2 == 0
-    n = len(conversations) // 2
-    examples = []
-    
-    for i in range(n):
-        user_msg = conversations[2*i]["value"]
-        assistant_msg = conversations[2*i+1]["value"]
-        examples.append({
-            "query": user_msg,
-            "response": assistant_msg
-        })
-    
-    return examples
-
-
-if __name__ == "__main__":
-    dataset = load_dataset("opencsg/smoltalk-chinese")
-    
-    process_dataset(
-        dataset_dict=dataset,
-        output_subdir="smoltalk-chinese",
-        process_func=process_func
-    )
\ No newline at end of file