diff --git a/supervised_finetuning/sft_firefly-1.1m-rephrased.py b/supervised_finetuning/sft_firefly-1.1m-rephrased.py new file mode 100644 index 0000000..673850e --- /dev/null +++ b/supervised_finetuning/sft_firefly-1.1m-rephrased.py @@ -0,0 +1,18 @@ +# Mxode/Firefly-1.1M-Rephrased +from datasets import load_dataset +from modules.utils import process_dataset + +def process_func(input_dict: dict): + instruction = input_dict["instruction"] + output = input_dict["output"] + return {"query": instruction, "response": output} + + +if __name__ == "__main__": + dataset = load_dataset("Mxode/Firefly-1.1M-Rephrased") + + process_dataset( + dataset_dict=dataset, + output_subdir="Firefly-1.1M-Rephrased", + process_func=process_func + ) diff --git a/supervised_finetuning/sft_small_talk.py b/supervised_finetuning/sft_small_talk.py deleted file mode 100644 index a0e9028..0000000 --- a/supervised_finetuning/sft_small_talk.py +++ /dev/null @@ -1,30 +0,0 @@ -# opencsg/smoltalk-chinese -from datasets import load_dataset -from modules.utils import process_dataset - - -def process_func(input_dict: dict): - conversations = input_dict["conversations"] - assert len(conversations) % 2 == 0 - n = len(conversations) // 2 - examples = [] - - for i in range(n): - user_msg = conversations[2*i]["value"] - assistant_msg = conversations[2*i+1]["value"] - examples.append({ - "query": user_msg, - "response": assistant_msg - }) - - return examples - - -if __name__ == "__main__": - dataset = load_dataset("opencsg/smoltalk-chinese") - - process_dataset( - dataset_dict=dataset, - output_subdir="smoltalk-chinese", - process_func=process_func - ) \ No newline at end of file