refactor(dataset): 重构数据处理脚本并支持自定义数据集

2025-07-15 12:18:58 +08:00
parent 1c7e49a631
commit bea4f83842
5 changed files with 25 additions and 15 deletions
@@ -0,0 +1,14 @@
+from utils import process_dataset
+
+if __name__ == "__main__":
+    max_chunk_num = 10
+    chunk_size = 1000000
+    item_size = max_chunk_num * chunk_size
+    
+    process_dataset(
+        dataset_name="chinese-cosmopedia",
+        output_subdir="chinese-wiki",
+        split=f"train[:{item_size}]",
+        max_chunk_num=max_chunk_num,
+        chunk_size=chunk_size,
+    )