refactor: remove inference redundancy and fix cache leaks
- drop Executor unused tokenizer field, _head_dim, stale metrics docstring - unify greedy sampling via SamplingPipeline.sample, drop top-level duplicate - drop Task.flush_remaining no-op and unreachable prompt-length branch - drop ProtocolHandler redundant chunks list (reuse body) - fix page_size=1 token-slot leak on task_free - clear _task_pages/_task_slots on alloc-failure paths - reset _bind_state on task_free to avoid stale steady-state reuse - remove unreachable contiguous branches in paged-only helpers
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
@@ -11,8 +10,6 @@ from tokenizers.decoders import DecodeStream
|
||||
from astrai.inference.core.metrics import MetricsCollector
|
||||
from astrai.tokenize.tokenizer import AutoTokenizer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
STOP = object()
|
||||
|
||||
|
||||
@@ -92,16 +89,6 @@ class Task:
|
||||
self._decoder = StreamDecoder(tokenizer)
|
||||
return self._decoder.push(self.output_ids[-1])
|
||||
|
||||
def flush_remaining(self, tokenizer: AutoTokenizer) -> str:
|
||||
"""Emit any text still buffered in the decoder.
|
||||
|
||||
With the Rust-native DecodeStream, the stream is always in a
|
||||
correct state — any completed text was already emitted by the
|
||||
last ``push``. A trailing incomplete multi-byte sequence has no
|
||||
valid text to emit, so this is a no-op.
|
||||
"""
|
||||
return ""
|
||||
|
||||
@property
|
||||
def next_pos(self) -> int:
|
||||
# The first output is sampled from prefill and enters KV on the next step.
|
||||
@@ -157,11 +144,6 @@ class TaskManager:
|
||||
if len(prompt_ids) > self.max_seq_len:
|
||||
prompt_ids = prompt_ids[-self.max_seq_len :]
|
||||
|
||||
if len(prompt_ids) > self.max_seq_len:
|
||||
if stream_callback:
|
||||
stream_callback(STOP)
|
||||
return task_id
|
||||
|
||||
if max_tokens is None:
|
||||
max_tokens = self.max_seq_len - len(prompt_ids)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user