refactor(utils): 重构数据处理脚本

2025-07-15 13:10:38 +08:00
parent e6f5b26a75
commit e77fb56c45
5 changed files with 20 additions and 21 deletions
@@ -1,8 +1,9 @@
+from datasets import load_dataset
 from utils import process_dataset

 if __name__ == "__main__":
+    dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")
    process_dataset(
-        dataset_name="HuggingFaceFW/fineweb",
+        dataset_dict=dataset,
        output_subdir="english-fineweb",
-        dataset_config="sample-10BT",
    )