refactor: harden inference cache state and attention dispatch
- split KVCache into phase-specific PrefillKVCache/DecodeKVCache types selected by start_pos - unify steady-state detection in TaskCacheManager - guard decode steady-state reuse with the cached task signature so recycled req slots cannot replay a prior generation's tokens and positions - collapse attention backend fwd_decode/fwd_prefill into a single subclass-owned forward with a shared _check_fwd guard - fix thread-safety gap in weight update and validate prefill inputs before KV allocation - centralize magic constants in InferenceConfig and align docs with behavior
This commit is contained in:
@@ -154,14 +154,18 @@ class _DummyBackend(AttentionBackend):
|
||||
def supports_call(self, q, kv_cache, attn_mask, is_causal, fwd) -> bool:
|
||||
return True
|
||||
|
||||
def fwd_decode(
|
||||
self, q, k, v, kv_cache=None, layer_id=0, attn_mask=None, is_causal=False
|
||||
):
|
||||
return q
|
||||
|
||||
def fwd_prefill(
|
||||
self, q, k, v, kv_cache=None, layer_id=0, attn_mask=None, is_causal=False
|
||||
def forward(
|
||||
self,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache=None,
|
||||
layer_id=0,
|
||||
attn_mask=None,
|
||||
is_causal=False,
|
||||
fwd=None,
|
||||
):
|
||||
self._check_fwd(fwd)
|
||||
return q
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user