From cd3a22c1acfa01aab08a2f912115af336fd17a96 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Thu, 2 Oct 2025 22:01:00 +0800
Subject: [PATCH] =?UTF-8?q?feat(utils):=20=E4=B8=BA=E7=BC=93=E5=AD=98?=
 =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=87=BD=E6=95=B0=E6=B7=BB=E5=8A=A0=20packin?=
 =?UTF-8?q?g=20=E5=92=8C=20padding=20=E5=8F=82=E6=95=B0=E6=94=AF=E6=8C=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 modules/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/utils.py b/modules/utils.py
index 619e40a..9e52c3c 100644
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -162,7 +162,7 @@ def get_dpo_processor(tokenizer: BpeTokenizer):
     return processor
         
 
-def cache_files(tokenizer, files, base_out_dir, cache_type):
+def cache_files(tokenizer, files, base_out_dir, cache_type, packing_size: int = -1, pad_value: int = 1):
     processor = None
     keys = []
     if cache_type == "pt":
@@ -177,7 +177,7 @@ def cache_files(tokenizer, files, base_out_dir, cache_type):
     else:
         raise ValueError("Invalid cache type")
     
-    dump_pkl_files(files, base_out_dir, processor, keys)
+    dump_pkl_files(files, base_out_dir, processor, keys, packing_size, pad_value)
                           
             
 def process_dataset(