merge remote main

tests/test_processors.py

@@ -13,21 +13,30 @@ from pipeline.processors import (
 
 
 class DummyTokenizer:
-    im_end = "<|im_end|>"
+    def __init__(self):
+        self._special_token_map = {}
+        self._chat_template = None
 
     def encode(self, text, add_special_tokens: bool = False):
         if isinstance(text, list):
             return [[ord(c) for c in item] for item in text]
         return [ord(c) for c in text]
 
-    def apply_chat_template(
-        self, messages, add_generation_prompt=True, tokenize=True
-    ):
+    def decode(self, tokens, skip_special_tokens=True):
+        return "".join(chr(t) for t in tokens)
+
+    def token_to_id(self, token: str):
+        return ord(token)
+
+    def set_chat_template(self, template):
+        self._chat_template = template
+
+    def apply_chat_template(self, messages, add_generation_prompt=True, tokenize=True):
         text = ""
         for m in messages:
-            text += f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n"
+            text += f"<｜im▁start｜>{m['role']}\n{m['content']}<｜im▁end｜>\n"
         if add_generation_prompt:
-            text += "<|im_start|>assistant\n"
+            text += "<｜im▁start｜>assistant\n"
         return self.encode(text) if tokenize else text