refactor(utils): 重构数据处理脚本

2025-07-15 13:10:38 +08:00
parent e6f5b26a75
commit e77fb56c45
5 changed files with 20 additions and 21 deletions
@@ -1,8 +1,10 @@
+from datasets import load_dataset
 from utils import process_dataset

 if __name__ == "__main__":
+    dataset = load_dataset("Blaze7451/enwiki_structured_content")
    process_dataset(
-        dataset_name="Blaze7451/enwiki_structured_content",
+        dataset_dict=dataset,
        output_subdir="english-wiki",
        max_chunk_size=5,
    )