ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add BELLE SFT export script, remove Firefly-1.1M-Rephrased

Browse Source
This commit is contained in:
ViperEkura
2026-07-17 23:15:18 +08:00
parent fec37545e2
commit e6787a2036
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/supervised_finetuning/sft_firefly-1.1m-rephrased.py → scripts/supervised_finetuning/sft_belle.py
+9 -3
View File
@@ -3,17 +3,23 @@ from pipeline import export_dataset
def process_func(input_dict: dict): def process_func(input_dict: dict):
instruction = input_dict["instruction"]
inp = input_dict.get("input", "")
if inp:
content = instruction + "\n" + inp
else:
content = instruction
return {"messages": [ return {"messages": [
{"role": "user", "content": input_dict["instruction"]}, {"role": "user", "content": content},
{"role": "assistant", "content": input_dict["output"]}, {"role": "assistant", "content": input_dict["output"]},
]} ]}
if __name__ == "__main__": if __name__ == "__main__":
dataset = load_dataset("Mxode/Firefly-1.1M-Rephrased") dataset = load_dataset("BelleGroup/train_2M_CN")
export_dataset( export_dataset(
dataset=dataset["train"], dataset=dataset["train"],
output_dir="./dataset", output_dir="./dataset",
output_prefix="Firefly-1.1M-Rephrased", output_prefix="belle-sft",
process_func=process_func, process_func=process_func,
) )