perf: reduce remaining per-step allocations
- hoist prefill qo_indptr into the workspace so CudaBackend.fwd_prefill does not rebuild it per layer - cache has_freq in SamplingBatchInfo to drop the per-step GPU any() sync - drop pin_memory host staging for input_ids; sync copy suffices for a small batch
This commit is contained in:
@@ -376,7 +376,7 @@ class CudaBackend(AttentionBackend):
|
||||
q_len = q.size(1)
|
||||
|
||||
kv_indptr = kv_cache.kv_indptr
|
||||
qo_indptr = torch.arange(b + 1, dtype=torch.int32, device=q.device) * q_len
|
||||
qo_indptr = kv_cache.qo_indptr
|
||||
|
||||
q_flat = q.reshape(b * q_len, q.size(2), q.size(3))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user