feat(tokenizer): 调整 BPE 分词器的字母表限制并新增用户与系统标识符属性
This commit is contained in:
@@ -44,7 +44,7 @@ class BpeTokenizer:
|
|||||||
trainer = BpeTrainer(
|
trainer = BpeTrainer(
|
||||||
vocab_size=detail_vocab_size,
|
vocab_size=detail_vocab_size,
|
||||||
min_frequency=min_freq,
|
min_frequency=min_freq,
|
||||||
limit_alphabet=detail_vocab_size // 2,
|
limit_alphabet=detail_vocab_size // 4,
|
||||||
max_token_length=18,
|
max_token_length=18,
|
||||||
special_tokens=self._control_tokens,
|
special_tokens=self._control_tokens,
|
||||||
show_progress=True,
|
show_progress=True,
|
||||||
@@ -109,3 +109,11 @@ class BpeTokenizer:
|
|||||||
@property
|
@property
|
||||||
def pad_id(self) -> int:
|
def pad_id(self) -> int:
|
||||||
return self._tokenizer.token_to_id("<pad>")
|
return self._tokenizer.token_to_id("<pad>")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def user_id(self) -> int:
|
||||||
|
return self._tokenizer.token_to_id("<|user|>")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def system_id(self) -> int:
|
||||||
|
return self._tokenizer.token_to_id("<|system|>")
|
||||||
Reference in New Issue
Block a user