ViperEkura
958df58f9d
- encode(str) single-thread, encode(List[str]) Rust parallel encode_batch - apply_chat_template accepts single Messages or List[Messages] for batch - add Message/Messages type aliases at module level
11 lines
245 B
Python
11 lines
245 B
Python
from astrai.tokenize.chat_template import ChatTemplate, MessageType
|
|
from astrai.tokenize.tokenizer import AutoTokenizer, Message, Messages
|
|
|
|
__all__ = [
|
|
"AutoTokenizer",
|
|
"ChatTemplate",
|
|
"MessageType",
|
|
"Message",
|
|
"Messages",
|
|
]
|