提交数据下载代码

2025-06-30 12:26:37 +08:00
commit fca1c541ba
5 changed files with 202 additions and 0 deletions
@@ -0,0 +1,5 @@
+from tokenizer import BpeTokenizer
+
+
+if __name__ == "__main__":
+    tokenzier = BpeTokenizer("tokenizer.json")