feat(project_structure): 重构项目目录并添加运行脚本

2025-08-01 17:41:49 +08:00
parent c4ce882d56
commit 2d1c4e9d6c
12 changed files with 75 additions and 2 deletions
@@ -0,0 +1,18 @@
+from datasets import load_dataset
+from utils import process_dataset
+
+if __name__ == "__main__":
+    max_chunk_num = 10
+    chunk_size = 1000000
+    
+    dataset = load_dataset(
+        "opencsg/chinese-cosmopedia", 
+        data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]}
+    )
+    
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="chinese-wiki",
+        max_chunk_num=max_chunk_num,
+        chunk_size=chunk_size,
+    )