fix: 修复特殊token 问题

This commit is contained in:
2026-04-02 16:16:02 +08:00
parent e01ec081b3
commit f44ad6912e
28 changed files with 334 additions and 163 deletions
+3 -1
View File
@@ -37,7 +37,9 @@ class SequencePacker:
identical chunk boundaries. Element-level correspondence is preserved.
"""
def __init__(self, pack_size: int, pad_value: int = 0, dtype: torch.dtype = torch.int32):
def __init__(
self, pack_size: int, pad_value: int = 0, dtype: torch.dtype = torch.int32
):
self.pack_size = pack_size
self.pad_value = pad_value
self.dtype = dtype