6 lines
111 B
Python
6 lines
111 B
Python
from tokenizer import BpeTokenizer
|
|
|
|
|
|
if __name__ == "__main__":
|
|
tokenzier = BpeTokenizer("tokenizer.json")
|