From 6238764047586bf5b82a56312dbb034262898d15 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Mon, 30 Jun 2025 21:45:51 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9C=AA=E5=88=9B=E5=BB=BA?=
 =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=B9=E7=9A=84bug?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 to_ids.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/to_ids.py b/to_ids.py
index f445eb0..0a7c447 100644
--- a/to_ids.py
+++ b/to_ids.py
@@ -25,6 +25,10 @@ def process_files(tokenizer: BpeTokenizer, files: List[str], base_out_dir):
     for file_path in tqdm(files, desc="Processing files", total=len(files)):
         out_file_name = os.path.basename(file_path).replace(".jsonl", ".pkl")
         out_file_path = os.path.join(base_out_dir, out_file_name)
+        
+        if not os.path.exists(out_file_path):
+            os.makedirs(os.path.dirname(out_file_path), exist_ok=True)
+            
         convert_to_ids(tokenizer, file_path, out_file_path)