From 1b49bfdea6e26ae2b3ddf7b8dc4b2fdded3dc44a Mon Sep 17 00:00:00 2001 From: ViperEkura <3081035982@qq.com> Date: Mon, 21 Jul 2025 11:23:53 +0800 Subject: [PATCH] =?UTF-8?q?chore:=20=E6=B7=BB=E5=8A=A0=20Chinese-Instruct?= =?UTF-8?q?=20=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=84=E7=90=86=E8=84=9A?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- chinese-instruct.py | 34 ++++++++++++++++++++++++++++++++++ utils.py | 1 + 2 files changed, 35 insertions(+) create mode 100644 chinese-instruct.py diff --git a/chinese-instruct.py b/chinese-instruct.py new file mode 100644 index 0000000..9e17967 --- /dev/null +++ b/chinese-instruct.py @@ -0,0 +1,34 @@ +# Mxode/Chinese-Reasoning-Distil-Data + +from datasets import DatasetDict +from datasets import load_dataset, concatenate_datasets +from utils import process_dataset + + +def build_prompt(query, response) -> str: + return f"<|user|> {query} <|system|> {response}\n" + + +def process_func(input_dict: dict): + query = input_dict["prompt"] + response = input_dict["response"] + return build_prompt(query, response) + + +if __name__ == "__main__": + all_data = ['stem_zh', 'infinity-instruct', 'firefly', 'magpie', 'dpsk-r1-distil', + 'coig-cqia', 'disc-law', 'neo_sft_phase2', 'chinese-medical', 'chinese-reasoning-distil', + 'psycho-10k-dpsk-r1', 'sof-c-zh', 'industryinstruction', 'Chinese-QA-AFAF'] + + datasets = [] + for subset in all_data: + ds = load_dataset("Mxode/Chinese-Instruct", name=subset) + datasets.append(ds["train"]) + + combined_dataset = concatenate_datasets(datasets) + + process_dataset( + dataset_dict=DatasetDict({"train": combined_dataset}), + output_subdir="chinese-instruct", + process_func=process_func, + ) \ No newline at end of file diff --git a/utils.py b/utils.py index f4701ab..392a507 100644 --- a/utils.py +++ b/utils.py @@ -76,6 +76,7 @@ def dump_pkl_files( cur_tensor = F.pad( cur_tensor, (0, packing_size - cur_tensor.numel()), + 'constant', tokenizer.pad_id ) packages.append(cur_tensor)