From fd99b9d852a19f57d4f8951e6f0a424c0edf9694 Mon Sep 17 00:00:00 2001 From: ViperEkura <3081035982@qq.com> Date: Thu, 4 Jun 2026 14:27:04 +0800 Subject: [PATCH] feat: add OpenHermes-2.5 SFT export script --- .../supervised_finetuning/sft_openhermes.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 scripts/supervised_finetuning/sft_openhermes.py diff --git a/scripts/supervised_finetuning/sft_openhermes.py b/scripts/supervised_finetuning/sft_openhermes.py new file mode 100644 index 0000000..b84d79c --- /dev/null +++ b/scripts/supervised_finetuning/sft_openhermes.py @@ -0,0 +1,22 @@ +from datasets import load_dataset +from pipeline import export_dataset + + +def process_func(input_dict: dict): + conversations = input_dict["conversations"] + examples = [] + for i in range(0, len(conversations) - 1, 2): + user_msg = conversations[i]["value"] + assistant_msg = conversations[i + 1]["value"] + examples.append({"query": user_msg, "response": assistant_msg}) + return examples + + +if __name__ == "__main__": + dataset = load_dataset("teknium/OpenHermes-2.5") + export_dataset( + dataset=dataset["train"], + output_dir="./dataset", + output_prefix="OpenHermes-2.5", + process_func=process_func, + )