diff --git a/belle_sft.py b/sft_belle.py similarity index 100% rename from belle_sft.py rename to sft_belle.py diff --git a/chinese-instruct.py b/sft_chinese_instruct.py similarity index 100% rename from chinese-instruct.py rename to sft_chinese_instruct.py diff --git a/sft_coder.py b/sft_coder.py new file mode 100644 index 0000000..3099f16 --- /dev/null +++ b/sft_coder.py @@ -0,0 +1,21 @@ +# inclusionAI/Ling-Coder-SFT +from datasets import load_dataset +from utils import process_dataset + + +def process_func(input_dict: dict) -> dict: + msg = input_dict["messages"] + query = msg[0]["content"] + history = msg[1]["content"] + return {"query": query, "response": history} + + +if __name__ == "__main__": + dataset = load_dataset("inclusionAI/Ling-Coder-SFT") + + process_dataset( + dataset_dict=dataset, + output_subdir="Ling-Coder-SFT", + process_func=process_func + ) + \ No newline at end of file diff --git a/utils.py b/utils.py index c76247b..86906b8 100644 --- a/utils.py +++ b/utils.py @@ -102,7 +102,7 @@ def dump_pkl_files( sequence = torch.cat(package[key]) output_package[key] = sequence - with open(out_file_path, "w") as f: + with open(out_file_path, "wb") as f: pkl.dump(output_package, f)