feat: 增加缓存处理

2026-04-08 20:54:14 +08:00
parent b0eff02446
commit ab5e207f42
3 changed files with 236 additions and 4 deletions
@@ -6,8 +6,7 @@ import json
 from pathlib import Path
 from typing import Dict, List, Optional, Union

-from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
-from tokenizers.models import BPE
+from tokenizers import Tokenizer

 from astrai.tokenize.chat_template import ChatTemplate