perf: use int32 paged KV indices
- store page-table, request-row, and cache-location indices as int32 - preserve CUDA graph replay with bit-exact logits and KV cache coverage - improve B=1 decode latency by 1-6% across 1K-32K contexts on L20
This commit is contained in:
@@ -55,8 +55,8 @@ struct AttentionParams {
|
||||
int mask_l_stride;
|
||||
|
||||
// Paged K/V addressing
|
||||
const int64_t* __restrict__ req_to_token; // [num_reqs, max_context_len]
|
||||
const int64_t* __restrict__ req_pool_indices; // [batch]
|
||||
const int* __restrict__ req_to_token; // [num_reqs, max_context_len]
|
||||
const int* __restrict__ req_pool_indices; // [batch]
|
||||
const int* __restrict__ kv_indptr; // [batch + 1]
|
||||
const int* __restrict__ qo_indptr; // [batch + 1] or nullptr for decode
|
||||
int max_context_len; // req_to_token stride (dim 1)
|
||||
|
||||
Reference in New Issue
Block a user