refactor: 重构 inference 模块架构,引入设计模式并分组文件
- 新增 protocol.py 协议层,Template Method 模式消除流/非流分支 45% 重复 - SSEBuilder 统一 SSE 构造,StopChecker 独立 stop_sequence 检测 - AnthropicHandler 追踪已产出文本,修复 stop 时重复 delta - server.py 路由从约 100 行缩减至 3 行 - 拆分为 core/(cache/executor/scheduler/task)和 api/(protocol/server) - 外部保持二级导入路径(from astrai.inference import Name) - 删除所有分隔线注释,代码按语义自然分组
This commit is contained in:
@@ -1,13 +1,40 @@
|
||||
"""Inference module for continuous batching.
|
||||
|
||||
Layers:
|
||||
- engine.py: Facade (InferenceEngine), Value Object (GenerationParams, GenerationRequest)
|
||||
- scheduler.py: Continuous-batching loop, Task state machine, TaskStatus enum
|
||||
- cache.py: PagedCache (page-table-indirected KV cache with alloc/free)
|
||||
- sampling.py: Strategy pattern (TemperatureStrategy, TopKStrategy, TopPStrategy)
|
||||
- server.py: FastAPI HTTP server (OpenAI-compatible endpoints)
|
||||
- core/: Core inference loop (cache, executor, scheduler, task)
|
||||
- api/: HTTP protocol handlers (OpenAI, Anthropic)
|
||||
- engine.py: Facade (InferenceEngine), Value Object (GenerationParams, GenerationRequest)
|
||||
- sample.py: Strategy pattern (TemperatureStrategy, TopKStrategy, TopPStrategy)
|
||||
"""
|
||||
|
||||
from astrai.inference.api import (
|
||||
AnthropicHandler,
|
||||
AnthropicMessage,
|
||||
ChatCompletionRequest,
|
||||
ChatMessage,
|
||||
MessagesRequest,
|
||||
OpenAIHandler,
|
||||
ProtocolHandler,
|
||||
SSEBuilder,
|
||||
StopChecker,
|
||||
StreamContext,
|
||||
app,
|
||||
run_server,
|
||||
)
|
||||
from astrai.inference.core import (
|
||||
STOP,
|
||||
CacheView,
|
||||
Executor,
|
||||
InferenceScheduler,
|
||||
PagedCache,
|
||||
PagePool,
|
||||
PrefixCache,
|
||||
Task,
|
||||
TaskManager,
|
||||
TaskStatus,
|
||||
TaskTable,
|
||||
page_hash,
|
||||
)
|
||||
from astrai.inference.engine import (
|
||||
GenerationParams,
|
||||
GenerationRequest,
|
||||
@@ -21,19 +48,26 @@ from astrai.inference.sample import (
|
||||
TopPStrategy,
|
||||
sample,
|
||||
)
|
||||
from astrai.inference.scheduler import InferenceScheduler
|
||||
from astrai.inference.task import STOP, Task, TaskStatus
|
||||
|
||||
__all__ = [
|
||||
# Engine / Requests
|
||||
"InferenceEngine",
|
||||
"GenerationRequest",
|
||||
"GenerationParams",
|
||||
# Scheduler
|
||||
# Core scheduler
|
||||
"InferenceScheduler",
|
||||
"Executor",
|
||||
"STOP",
|
||||
"Task",
|
||||
"TaskManager",
|
||||
"TaskStatus",
|
||||
# Core cache
|
||||
"CacheView",
|
||||
"PagedCache",
|
||||
"PagePool",
|
||||
"PrefixCache",
|
||||
"TaskTable",
|
||||
"page_hash",
|
||||
# Sampling (Strategy pattern)
|
||||
"sample",
|
||||
"BaseSamplingStrategy",
|
||||
@@ -41,4 +75,18 @@ __all__ = [
|
||||
"TopKStrategy",
|
||||
"TopPStrategy",
|
||||
"SamplingPipeline",
|
||||
# Protocol
|
||||
"ProtocolHandler",
|
||||
"SSEBuilder",
|
||||
"StopChecker",
|
||||
"StreamContext",
|
||||
"AnthropicHandler",
|
||||
"OpenAIHandler",
|
||||
# Server
|
||||
"ChatMessage",
|
||||
"ChatCompletionRequest",
|
||||
"AnthropicMessage",
|
||||
"MessagesRequest",
|
||||
"app",
|
||||
"run_server",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user