feat: add per-task throughput and latency metrics

- extract TaskTiming + MetricsCollector out of Task/TaskManager
- unify prefill/decode timing into single record() context manager
- expose avg_ttft_ms, avg_decode_tps, avg_e2e_latency_ms via /stats
This commit is contained in:
2026-08-08 12:10:06 +08:00
parent c52a2487ae
commit cb60713a72
4 changed files with 263 additions and 11 deletions
+3
View File
@@ -10,6 +10,7 @@ from astrai.inference.core.cache import (
page_hash,
)
from astrai.inference.core.executor import Executor
from astrai.inference.core.metrics import MetricsCollector, TaskTiming
from astrai.inference.core.scheduler import InferenceScheduler
from astrai.inference.core.task import STOP, Task, TaskManager, TaskStatus
@@ -23,6 +24,8 @@ __all__ = [
"page_hash",
"Executor",
"InferenceScheduler",
"MetricsCollector",
"TaskTiming",
"STOP",
"Task",
"TaskManager",