feat(dataset): 更新数据集处理和保存逻辑

2025-07-20 11:21:18 +08:00
parent 365086f102
commit fcb5c2bcc8
3 changed files with 17 additions and 2 deletions
@@ -38,7 +38,7 @@ def dump_pkl_files(
 ):  
    def process_line(line: str) -> Tensor:
            line = json.loads(line)[key]
-            processed_line = encoder(line)
+            processed_line = encoder(line) if encoder else line
            ids = tokenizer.encode(processed_line)
            arrow = torch.tensor(ids, dtype=torch.int32)
            return arrow