diff --git a/chinese-instruct.py b/chinese-instruct.py new file mode 100644 index 0000000..9e17967 --- /dev/null +++ b/chinese-instruct.py @@ -0,0 +1,34 @@ +# Mxode/Chinese-Reasoning-Distil-Data + +from datasets import DatasetDict +from datasets import load_dataset, concatenate_datasets +from utils import process_dataset + + +def build_prompt(query, response) -> str: + return f"<|user|> {query} <|system|> {response}\n" + + +def process_func(input_dict: dict): + query = input_dict["prompt"] + response = input_dict["response"] + return build_prompt(query, response) + + +if __name__ == "__main__": + all_data = ['stem_zh', 'infinity-instruct', 'firefly', 'magpie', 'dpsk-r1-distil', + 'coig-cqia', 'disc-law', 'neo_sft_phase2', 'chinese-medical', 'chinese-reasoning-distil', + 'psycho-10k-dpsk-r1', 'sof-c-zh', 'industryinstruction', 'Chinese-QA-AFAF'] + + datasets = [] + for subset in all_data: + ds = load_dataset("Mxode/Chinese-Instruct", name=subset) + datasets.append(ds["train"]) + + combined_dataset = concatenate_datasets(datasets) + + process_dataset( + dataset_dict=DatasetDict({"train": combined_dataset}), + output_subdir="chinese-instruct", + process_func=process_func, + ) \ No newline at end of file diff --git a/utils.py b/utils.py index f4701ab..392a507 100644 --- a/utils.py +++ b/utils.py @@ -76,6 +76,7 @@ def dump_pkl_files( cur_tensor = F.pad( cur_tensor, (0, packing_size - cur_tensor.numel()), + 'constant', tokenizer.pad_id ) packages.append(cur_tensor)