feat: add per-task throughput and latency metrics
- extract TaskTiming + MetricsCollector out of Task/TaskManager - unify prefill/decode timing into single record() context manager - expose avg_ttft_ms, avg_decode_tps, avg_e2e_latency_ms via /stats
This commit is contained in:
@@ -10,6 +10,7 @@ from astrai.inference.core.cache import (
|
||||
page_hash,
|
||||
)
|
||||
from astrai.inference.core.executor import Executor
|
||||
from astrai.inference.core.metrics import MetricsCollector, TaskTiming
|
||||
from astrai.inference.core.scheduler import InferenceScheduler
|
||||
from astrai.inference.core.task import STOP, Task, TaskManager, TaskStatus
|
||||
|
||||
@@ -23,6 +24,8 @@ __all__ = [
|
||||
"page_hash",
|
||||
"Executor",
|
||||
"InferenceScheduler",
|
||||
"MetricsCollector",
|
||||
"TaskTiming",
|
||||
"STOP",
|
||||
"Task",
|
||||
"TaskManager",
|
||||
|
||||
Reference in New Issue
Block a user