ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(utils): 优化数据集处理中的分块逻辑

Browse Source
This commit is contained in:
ViperEkura
2025-07-14 14:52:07 +08:00
parent 7e53d35099
commit 4cf9681ebb
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils.py
+1 -1
View File
@@ -65,7 +65,7 @@ def process_dataset(
total_samples = len(train_dataset) total_samples = len(train_dataset)
num_chunks = (total_samples // chunk_size) + 1 num_chunks = (total_samples // chunk_size) + 1
lim_chunks = max_chunk_size if max_chunk_size else num_chunks lim_chunks = min(max_chunk_size, num_chunks) if max_chunk_size else num_chunks
script_dir = os.path.dirname(os.path.abspath(__file__)) script_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(script_dir, "dataset", output_subdir) output_dir = os.path.join(script_dir, "dataset", output_subdir)