From 2ccb26add401c6fb1806a6d2cf4e0db578feb686 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Thu, 2 Oct 2025 22:00:52 +0800
Subject: [PATCH] =?UTF-8?q?feat(tokenizer):=20=E8=B0=83=E6=95=B4=20BPE=20?=
 =?UTF-8?q?=E5=88=86=E8=AF=8D=E5=99=A8=E7=9A=84=E5=AD=97=E6=AF=8D=E8=A1=A8?=
 =?UTF-8?q?=E9=99=90=E5=88=B6=E5=B9=B6=E6=96=B0=E5=A2=9E=E7=94=A8=E6=88=B7?=
 =?UTF-8?q?=E4=B8=8E=E7=B3=BB=E7=BB=9F=E6=A0=87=E8=AF=86=E7=AC=A6=E5=B1=9E?=
 =?UTF-8?q?=E6=80=A7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 modules/tokenizer.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/modules/tokenizer.py b/modules/tokenizer.py
index 107f001..65e06b4 100644
--- a/modules/tokenizer.py
+++ b/modules/tokenizer.py
@@ -44,7 +44,7 @@ class BpeTokenizer:
         trainer = BpeTrainer(
             vocab_size=detail_vocab_size,
             min_frequency=min_freq,
-            limit_alphabet=detail_vocab_size // 2,
+            limit_alphabet=detail_vocab_size // 4,
             max_token_length=18,
             special_tokens=self._control_tokens,
             show_progress=True,
@@ -109,3 +109,11 @@ class BpeTokenizer:
     @property
     def pad_id(self) -> int:
         return self._tokenizer.token_to_id("<pad>")
+    
+    @property
+    def user_id(self) -> int:
+        return self._tokenizer.token_to_id("<|user|>")
+    
+    @property
+    def system_id(self) -> int:
+        return self._tokenizer.token_to_id("<|system|>")
\ No newline at end of file