refactor: remove inference redundancy and fix cache leaks

- drop Executor unused tokenizer field, _head_dim, stale metrics docstring
- unify greedy sampling via SamplingPipeline.sample, drop top-level duplicate
- drop Task.flush_remaining no-op and unreachable prompt-length branch
- drop ProtocolHandler redundant chunks list (reuse body)
- fix page_size=1 token-slot leak on task_free
- clear _task_pages/_task_slots on alloc-failure paths
- reset _bind_state on task_free to avoid stale steady-state reuse
- remove unreachable contiguous branches in paged-only helpers
This commit is contained in:
2026-08-08 21:45:01 +08:00
parent d9240ab149
commit ca50fe4721
7 changed files with 12 additions and 52 deletions
-18
View File
@@ -1,4 +1,3 @@
import logging
import threading
import time
import uuid
@@ -11,8 +10,6 @@ from tokenizers.decoders import DecodeStream
from astrai.inference.core.metrics import MetricsCollector
from astrai.tokenize.tokenizer import AutoTokenizer
logger = logging.getLogger(__name__)
STOP = object()
@@ -92,16 +89,6 @@ class Task:
self._decoder = StreamDecoder(tokenizer)
return self._decoder.push(self.output_ids[-1])
def flush_remaining(self, tokenizer: AutoTokenizer) -> str:
"""Emit any text still buffered in the decoder.
With the Rust-native DecodeStream, the stream is always in a
correct state — any completed text was already emitted by the
last ``push``. A trailing incomplete multi-byte sequence has no
valid text to emit, so this is a no-op.
"""
return ""
@property
def next_pos(self) -> int:
# The first output is sampled from prefill and enters KV on the next step.
@@ -157,11 +144,6 @@ class TaskManager:
if len(prompt_ids) > self.max_seq_len:
prompt_ids = prompt_ids[-self.max_seq_len :]
if len(prompt_ids) > self.max_seq_len:
if stream_callback:
stream_callback(STOP)
return task_id
if max_tokens is None:
max_tokens = self.max_seq_len - len(prompt_ids)
else: