refactor(utils): 重构数据处理脚本

2025-07-15 13:10:38 +08:00
parent e6f5b26a75
commit e77fb56c45
5 changed files with 20 additions and 21 deletions
@@ -1,7 +1,9 @@
+from datasets import load_dataset
 from utils import process_dataset

 if __name__ == "__main__":
+    dataset = load_dataset("shjwudp/chinese-c4")
    process_dataset(
-        dataset_name="shjwudp/chinese-c4",
+        dataset_dict=dataset,
        output_subdir="chinese-c4"
    )