refactor: 修改项目结构

2026-03-21 21:25:59 +08:00
parent 5b47ab1dea
commit 909f1b400c
32 changed files with 535 additions and 703 deletions
@@ -0,0 +1,20 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(input_dict: dict):
+    return {
+        "prompt": input_dict["prompt"],
+        "chosen": input_dict["chosen"],
+        "rejected": input_dict["rejected"],
+    }
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("wenbopan/Chinese-dpo-pairs")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="Chinese-dpo-pairs",
+        process_func=process_func,
+    )