refactor(dataset): 重构数据处理脚本并支持自定义数据集

2025-07-15 12:18:58 +08:00
parent 1c7e49a631
commit bea4f83842
5 changed files with 25 additions and 15 deletions
@@ -93,8 +93,8 @@ class BpeTokenizer:
            else:
                return [encoding.tokens for encoding in encodings]

-    def decode(self, tokens: List[int]) -> str:
-        return self._tokenizer.decode(tokens)
+    def decode(self, tokens: List[int], skip_special_tokens=True) -> str:
+        return self._tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens)
    
    def __len__(self) -> int:
        return self._tokenizer.get_vocab_size()