feat(project_structure): 重构项目目录并添加运行脚本

2025-08-01 17:41:49 +08:00
parent c4ce882d56
commit 2d1c4e9d6c
12 changed files with 75 additions and 2 deletions
@@ -0,0 +1,9 @@
+from datasets import load_dataset
+from utils import process_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="english-fineweb",
+    )