ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: padding 默认值改为 0，避免 loss_mask 误计 padding 位置

Browse Source
This commit is contained in:
ViperEkura
2026-05-15 16:59:42 +08:00
parent cb6bfcb976
commit 625695fd71
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline/io/export.py
+1 -1
View File
@@ -83,7 +83,7 @@ def cache_jsonl(
processor: BaseProcessor, processor: BaseProcessor,
*, *,
pack_size: int = -1, pack_size: int = -1,
pad_value: int = 1, pad_value: int = 0,
) -> List[str]: ) -> List[str]:
"""Tokenize JSONL files and pack them into HDF5 storage. """Tokenize JSONL files and pack them into HDF5 storage.
scripts/cache_h5.py
+1 -1
View File
@@ -51,7 +51,7 @@ def main():
help="Pack size, <=0 to disable (default: -1)", help="Pack size, <=0 to disable (default: -1)",
) )
parser.add_argument( parser.add_argument(
"--pad-value", type=int, default=1, help="Padding value (default: 1)" "--pad-value", type=int, default=0, help="Padding value (default: 0)"
) )
parser.add_argument( parser.add_argument(
"--log-level", "--log-level",