refactor: 重构 cache 和 inference 参数体系,分离存储与分配

- 合并 GenerationRequest/GenerationParams,统一 max_tokens 参数名
- PagePool/PrefixCache 分离为 Allocator + PrefixCache + PagePool
- 拆分 KV 存储为独立 Storage 类,PagedCache → KVCache,CacheView → KvcacheView
- Allocator.inc_ref 移除 LRU 防止竞争,Storage.write 增加负页防御
- Allocator/PrefixCache/TaskTable 加 threading.Lock 保证线程安全
- server.py uvicorn.run 改为传 app 对象修复导入错误
- benchmark.py 适配 KVCache 新 API
This commit is contained in:
2026-05-14 20:05:08 +08:00
parent 18fe6e9339
commit 205b40bd28
13 changed files with 394 additions and 351 deletions
+9 -7
View File
@@ -3,7 +3,7 @@
Layers:
- core/: Core inference loop (cache, executor, scheduler, task)
- api/: HTTP protocol handlers (OpenAI, Anthropic)
- engine.py: Facade (InferenceEngine), Value Object (GenerationParams, GenerationRequest)
- engine.py: Facade (InferenceEngine), Value Object (GenerationRequest)
- sample.py: Strategy pattern (TemperatureStrategy, TopKStrategy, TopPStrategy)
"""
@@ -22,12 +22,14 @@ from astrai.inference.api import (
)
from astrai.inference.core import (
STOP,
CacheView,
Allocator,
Executor,
InferenceScheduler,
PagedCache,
KVCache,
KvcacheView,
PagePool,
PrefixCache,
Storage,
Task,
TaskManager,
TaskStatus,
@@ -35,7 +37,6 @@ from astrai.inference.core import (
page_hash,
)
from astrai.inference.engine import (
GenerationParams,
GenerationRequest,
InferenceEngine,
)
@@ -52,7 +53,6 @@ __all__ = [
# Engine / Requests
"InferenceEngine",
"GenerationRequest",
"GenerationParams",
# Core scheduler
"InferenceScheduler",
"Executor",
@@ -61,10 +61,12 @@ __all__ = [
"TaskManager",
"TaskStatus",
# Core cache
"CacheView",
"PagedCache",
"Allocator",
"KVCache",
"KvcacheView",
"PagePool",
"PrefixCache",
"Storage",
"TaskTable",
"page_hash",
# Sampling (Strategy pattern)