diff --git a/scripts/supervised_finetuning/sft_firefly-1.1m-rephrased.py b/scripts/supervised_finetuning/sft_belle.py similarity index 53% rename from scripts/supervised_finetuning/sft_firefly-1.1m-rephrased.py rename to scripts/supervised_finetuning/sft_belle.py index 76d6e90..f92dac5 100644 --- a/scripts/supervised_finetuning/sft_firefly-1.1m-rephrased.py +++ b/scripts/supervised_finetuning/sft_belle.py @@ -3,17 +3,23 @@ from pipeline import export_dataset def process_func(input_dict: dict): + instruction = input_dict["instruction"] + inp = input_dict.get("input", "") + if inp: + content = instruction + "\n" + inp + else: + content = instruction return {"messages": [ - {"role": "user", "content": input_dict["instruction"]}, + {"role": "user", "content": content}, {"role": "assistant", "content": input_dict["output"]}, ]} if __name__ == "__main__": - dataset = load_dataset("Mxode/Firefly-1.1M-Rephrased") + dataset = load_dataset("BelleGroup/train_2M_CN") export_dataset( dataset=dataset["train"], output_dir="./dataset", - output_prefix="Firefly-1.1M-Rephrased", + output_prefix="belle-sft", process_func=process_func, )