refactor(dataset processing): 重构数据集处理流程并添加新功能

2025-07-14 10:57:23 +08:00
parent 7591a231df
commit d8643d11dd
3 changed files with 22 additions and 14 deletions
@@ -0,0 +1,7 @@
+from utils import process_dataset
+
+if __name__ == "__main__":
+    process_dataset(
+        dataset_name="Blaze7451/enwiki_structured_content",
+        output_subdir="english-wiki"
+    )