chore: 添加 Chinese-Instruct 数据集处理脚本

2025-07-21 11:23:53 +08:00
parent fcb5c2bcc8
commit 1b49bfdea6
2 changed files with 35 additions and 0 deletions
@@ -76,6 +76,7 @@ def dump_pkl_files(
                        cur_tensor = F.pad(
                            cur_tensor, 
                            (0, packing_size - cur_tensor.numel()),
+                            'constant',
                            tokenizer.pad_id
                        )
                        packages.append(cur_tensor)