refactor: extract shared steady-state increment detection

- add _BindState dataclass and _is_steady_increment() to cache.py
- replace _bind_sig/_bind_seq_lens dual fields with single _bind_state
- replace DecodeSteadyState bare tuple with named dataclass
- use _is_steady_increment() in both PagePool.bind_tasks and Executor.execute_decode
This commit is contained in:
2026-08-07 23:00:25 +08:00
parent 02469887f5
commit 184fbbce5c
2 changed files with 57 additions and 30 deletions
+24 -14
View File
@@ -14,7 +14,7 @@ from astrai.extension.attention_backend import (
attn_backend,
get_backend,
)
from astrai.inference.core.cache import PagePool
from astrai.inference.core.cache import PagePool, _is_steady_increment
from astrai.inference.core.graph import CudaGraphContext
from astrai.inference.core.task import Task
from astrai.inference.core.workspace import InferenceWorkspace
@@ -54,6 +54,19 @@ class SamplingBatchInfo:
has_freq: bool # any frequency_penalty != 0 (avoids per-step GPU .any())
@dataclass
class DecodeSteadyState:
"""Cached decode metadata for the steady-state case.
When the same ordered task set decodes one token per step, sampling
params and task signature are reused; only positions advance by 1.
"""
task_sig: tuple
positions: list[int]
sampling_info: SamplingBatchInfo
def _build_sampling_batch_info(tasks: List[Task], device) -> SamplingBatchInfo:
pin = str(device).startswith("cuda")
freq_penalties = torch.tensor(
@@ -165,11 +178,10 @@ class Executor:
self.device = device or next(model.parameters()).device
self.dtype = dtype or next(model.parameters()).dtype
# Per-step decode cache for the steady-state case where the same
# ordered task set decodes one token per step. Sampling params are
# constant across steps; position_ids grows by exactly 1. Single-slot:
# any task-set change is a cache miss.
self._decode_cache: Optional[tuple] = None
# Per-step decode cache for the steady-state case (same ordered
# task set decodes one token per step). Sampling params stay
# constant; only positions advance.
self._decode_cache: Optional[DecodeSteadyState] = None
# Pre-allocated fixed-shape buffers for the decode hot path
# (input_ids, decode mask, KV bind metadata). Eagerly sized at init
@@ -340,20 +352,18 @@ class Executor:
sig = tuple(task_ids)
cached = self._decode_cache
if (
cached is not None
and cached[0] == sig
and cur_positions == [p + 1 for p in cached[1]]
):
info = cached[2]
prev_sig = cached.task_sig if cached is not None else None
prev_pos = cached.positions if cached is not None else None
if _is_steady_increment(prev_sig, prev_pos, sig, cur_positions):
info = cached.sampling_info
ws.position_ids[:b] += 1
self._decode_cache = (sig, cur_positions, info)
self._decode_cache = DecodeSteadyState(sig, cur_positions, info)
else:
info = _build_sampling_batch_info(tasks, self.device)
ws.position_ids[:b].copy_(
torch.tensor(cur_positions, dtype=torch.long, device=self.device)
)
self._decode_cache = (sig, cur_positions, info)
self._decode_cache = DecodeSteadyState(sig, cur_positions, info)
total_len = max(cur_positions) + 1
input_mask = ws.decode_mask(ws.position_ids[:b], total_len)