refactor: 重构 cache 和 inference 参数体系,分离存储与分配

- 合并 GenerationRequest/GenerationParams,统一 max_tokens 参数名
- PagePool/PrefixCache 分离为 Allocator + PrefixCache + PagePool
- 拆分 KV 存储为独立 Storage 类,PagedCache → KVCache,CacheView → KvcacheView
- Allocator.inc_ref 移除 LRU 防止竞争,Storage.write 增加负页防御
- Allocator/PrefixCache/TaskTable 加 threading.Lock 保证线程安全
- server.py uvicorn.run 改为传 app 对象修复导入错误
- benchmark.py 适配 KVCache 新 API
This commit is contained in:
2026-05-14 20:05:08 +08:00
parent 18fe6e9339
commit 205b40bd28
13 changed files with 394 additions and 351 deletions
+3 -3
View File
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Union
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from astrai.inference.engine import GenerationParams, InferenceEngine
from astrai.inference.engine import InferenceEngine
def _sse_event(data: Dict[str, Any], event: Optional[str] = None) -> str:
@@ -143,13 +143,13 @@ class ProtocolHandler(ABC):
prompt_tokens=self._count_prompt_tokens(),
)
params = GenerationParams(
agen = self.engine.generate_async(
prompt=self.build_prompt(),
max_tokens=self.request.max_tokens,
temperature=self.request.temperature,
top_p=self.request.top_p,
top_k=self.request.top_k,
)
agen = self.engine.generate_async(prompt=self.build_prompt(), params=params)
if self.request.stream:
return self._handle_stream(agen, ctx)
+1 -2
View File
@@ -160,8 +160,7 @@ def run_server(
"max_batch_size": max_batch_size,
}
uvicorn.run(
"astrai.inference.server:app",
app,
host=host,
port=port,
reload=reload,
)