refactor: remove inference redundancy and fix cache leaks
- drop Executor unused tokenizer field, _head_dim, stale metrics docstring - unify greedy sampling via SamplingPipeline.sample, drop top-level duplicate - drop Task.flush_remaining no-op and unreachable prompt-length branch - drop ProtocolHandler redundant chunks list (reuse body) - fix page_size=1 token-slot leak on task_free - clear _task_pages/_task_slots on alloc-failure paths - reset _bind_state on task_free to avoid stale steady-state reuse - remove unreachable contiguous branches in paged-only helpers
This commit is contained in:
@@ -12,7 +12,7 @@ class TaskTiming:
|
||||
"""Timestamp snapshots and computed metrics for one generation task.
|
||||
|
||||
Created by :class:`MetricsCollector` at task-registration time;
|
||||
updated via ``prefill_scope`` / ``mark_finished``.
|
||||
updated via ``record`` / ``mark_finished``.
|
||||
"""
|
||||
|
||||
task_id: str
|
||||
@@ -27,8 +27,6 @@ class TaskTiming:
|
||||
|
||||
# derived metrics
|
||||
|
||||
# derived metrics
|
||||
|
||||
@property
|
||||
def queue_wait_ms(self) -> Optional[float]:
|
||||
if self.prefill_start_time is not None:
|
||||
@@ -116,7 +114,7 @@ class MetricsCollector:
|
||||
metrics = MetricsCollector()
|
||||
metrics.register(task_id, arrival_time)
|
||||
|
||||
with metrics.prefill_scope(task_ids):
|
||||
with metrics.record(task_ids, "prefill"):
|
||||
run_prefill(...)
|
||||
|
||||
metrics.mark_finished(task_id, input_tokens, output_tokens)
|
||||
|
||||
Reference in New Issue
Block a user