reafactor: 重构项目

2026-03-30 20:58:51 +08:00
parent f67bad0d8b
commit 35963bcb08
29 changed files with 1395 additions and 1234 deletions
@@ -17,12 +17,12 @@ if __name__ == "__main__":
        'psycho-10k-dpsk-r1', 'sof-c-zh', 'industryinstruction', 'Chinese-QA-AFAF',
    ]

-    datasets = []
+    dataset_list = []
    for subset in all_data:
        ds = load_dataset("Mxode/Chinese-Instruct", name=subset)
-        datasets.append(ds["train"])
+        dataset_list.append(ds["train"])

-    combined_dataset = concatenate_datasets(datasets)
+    combined_dataset = concatenate_datasets(dataset_list)
    export_dataset(
        dataset=combined_dataset,
        output_dir="./dataset",