perf: reduce remaining per-step allocations

- hoist prefill qo_indptr into the workspace so CudaBackend.fwd_prefill does not rebuild it per layer
- cache has_freq in SamplingBatchInfo to drop the per-step GPU any() sync
- drop pin_memory host staging for input_ids; sync copy suffices for a small batch
This commit is contained in:
2026-08-03 01:10:06 +08:00
parent a03504a280
commit d0e5d910de
4 changed files with 31 additions and 14 deletions
+1 -1
View File
@@ -376,7 +376,7 @@ class CudaBackend(AttentionBackend):
q_len = q.size(1)
kv_indptr = kv_cache.kv_indptr
qo_indptr = torch.arange(b + 1, dtype=torch.int32, device=q.device) * q_len
qo_indptr = kv_cache.qo_indptr
q_flat = q.reshape(b * q_len, q.size(2), q.size(3))