From d9ef711e2846def4910727a20bd8ae9be5948931 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Mon, 4 Aug 2025 15:18:46 +0800
Subject: [PATCH] =?UTF-8?q?refactor(pretrain):=20=E6=9B=B4=E6=96=B0?=
 =?UTF-8?q?=E9=A2=84=E8=AE=AD=E7=BB=83=E6=95=B0=E6=8D=AE=E9=9B=86=E8=BE=93?=
 =?UTF-8?q?=E5=87=BA=E7=9B=AE=E5=BD=95=E5=90=8D=E7=A7=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 reforce_learning/dpp_chinese_dpo_pairs.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 reforce_learning/dpp_chinese_dpo_pairs.py

diff --git a/reforce_learning/dpp_chinese_dpo_pairs.py b/reforce_learning/dpp_chinese_dpo_pairs.py
new file mode 100644
index 0000000..6f60453
--- /dev/null
+++ b/reforce_learning/dpp_chinese_dpo_pairs.py
@@ -0,0 +1,18 @@
+from datasets import load_dataset
+from modules.utils import process_dataset
+
+def process_func(input_dict: dict):
+    return {
+        "promopt": input_dict["prompt"],
+        "chosen": input_dict["chosen"],
+        "rejected": input_dict["rejected"]
+    }
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("wenbopan/Chinese-dpo-pairs")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="chinese-c4-pretrain",
+        process_func=process_func
+    )
\ No newline at end of file