feat(dataset): 更新数据集处理和保存逻辑

2025-07-20 11:21:18 +08:00
parent 365086f102
commit fcb5c2bcc8
3 changed files with 17 additions and 2 deletions
@@ -0,0 +1,15 @@
+from utils import dump_pkl_files, fetch_files
+from tokenizer import BpeTokenizer
+import os
+
+if __name__ == "__main__":
+    tokenizer = BpeTokenizer("tokenizer.json")
+    base_dir = [
+        os.path.join("dataset", "belle-sft"),
+    ]
+    base_out_dir = "pkl_output"
+    files = []
+    for dir_path in base_dir:
+        files.extend(fetch_files(dir_path))
+    
+    dump_pkl_files(tokenizer, files, base_out_dir, packing_size=2048)