refactor(supervised_finetuning): 移除 smoltalk-chinese 数据集处理脚本

2025-08-10 11:30:08 +08:00
parent ce263985d9
commit 6a259cab0d
2 changed files with 18 additions and 30 deletions
@@ -0,0 +1,18 @@
+# Mxode/Firefly-1.1M-Rephrased
+from datasets import load_dataset
+from modules.utils import process_dataset
+
+def process_func(input_dict: dict):
+    instruction = input_dict["instruction"]
+    output = input_dict["output"]
+    return {"query": instruction, "response": output}
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("Mxode/Firefly-1.1M-Rephrased")
+    
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="Firefly-1.1M-Rephrased",
+        process_func=process_func
+    )