diff --git a/reforce_learning/dpp_chinese_dpo_pairs.py b/reforce_learning/dpp_chinese_dpo_pairs.py new file mode 100644 index 0000000..6f60453 --- /dev/null +++ b/reforce_learning/dpp_chinese_dpo_pairs.py @@ -0,0 +1,18 @@ +from datasets import load_dataset +from modules.utils import process_dataset + +def process_func(input_dict: dict): + return { + "promopt": input_dict["prompt"], + "chosen": input_dict["chosen"], + "rejected": input_dict["rejected"] + } + + +if __name__ == "__main__": + dataset = load_dataset("wenbopan/Chinese-dpo-pairs") + process_dataset( + dataset_dict=dataset, + output_subdir="chinese-c4-pretrain", + process_func=process_func + ) \ No newline at end of file