refactor(pretrain): 更新预训练数据集输出目录名称

2025-08-04 15:18:46 +08:00
parent 241e24c400
commit d9ef711e28
1 changed files with 18 additions and 0 deletions
@@ -0,0 +1,18 @@
+from datasets import load_dataset
+from modules.utils import process_dataset
+
+def process_func(input_dict: dict):
+    return {
+        "promopt": input_dict["prompt"],
+        "chosen": input_dict["chosen"],
+        "rejected": input_dict["rejected"]
+    }
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("wenbopan/Chinese-dpo-pairs")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="chinese-c4-pretrain",
+        process_func=process_func
+    )