refactor: 修改项目结构

2026-03-21 21:25:59 +08:00
parent 5b47ab1dea
commit 909f1b400c
32 changed files with 535 additions and 703 deletions
@@ -0,0 +1,31 @@
+from datasets import load_dataset, concatenate_datasets
+from pipeline import export_dataset, TextNormalizer
+
+normalizer = TextNormalizer()
+
+
+def process_func(input_dict: dict):
+    query = input_dict["prompt"] if input_dict["prompt"] else ""
+    resp = input_dict["response"] if input_dict["response"] else ""
+    return {"query": normalizer.normalize(query), "response": normalizer.normalize(resp)}
+
+
+if __name__ == "__main__":
+    all_data = [
+        'stem_zh', 'infinity-instruct', 'firefly', 'magpie', 'dpsk-r1-distil',
+        'coig-cqia', 'disc-law', 'neo_sft_phase2', 'chinese-medical', 'chinese-reasoning-distil',
+        'psycho-10k-dpsk-r1', 'sof-c-zh', 'industryinstruction', 'Chinese-QA-AFAF',
+    ]
+
+    datasets = []
+    for subset in all_data:
+        ds = load_dataset("Mxode/Chinese-Instruct", name=subset)
+        datasets.append(ds["train"])
+
+    combined_dataset = concatenate_datasets(datasets)
+    export_dataset(
+        dataset=combined_dataset,
+        output_dir="./dataset",
+        output_prefix="chinese-instruct-sft",
+        process_func=process_func,
+    )