refactor: harden inference cache state and attention dispatch

- split KVCache into phase-specific PrefillKVCache/DecodeKVCache types selected by start_pos
- unify steady-state detection in TaskCacheManager
- guard decode steady-state reuse with the cached task signature so recycled req slots cannot replay a prior generation's tokens and positions
- collapse attention backend fwd_decode/fwd_prefill into a single subclass-owned forward with a shared _check_fwd guard
- fix thread-safety gap in weight update and validate prefill inputs before KV allocation
- centralize magic constants in InferenceConfig and align docs with behavior
This commit is contained in:
2026-09-04 14:28:04 +08:00
parent e13fe53475
commit ae7fc3059a
12 changed files with 244 additions and 152 deletions
+11 -7
View File
@@ -154,14 +154,18 @@ class _DummyBackend(AttentionBackend):
def supports_call(self, q, kv_cache, attn_mask, is_causal, fwd) -> bool:
return True
def fwd_decode(
self, q, k, v, kv_cache=None, layer_id=0, attn_mask=None, is_causal=False
):
return q
def fwd_prefill(
self, q, k, v, kv_cache=None, layer_id=0, attn_mask=None, is_causal=False
def forward(
self,
q,
k,
v,
kv_cache=None,
layer_id=0,
attn_mask=None,
is_causal=False,
fwd=None,
):
self._check_fwd(fwd)
return q