perf: pack gqa q-heads per prefill block to reuse kv tiles

- pack HB = min(G, WARPS) q heads per block; K/V tiles stream once per block instead of once per q head
- G=1 keeps the old grid; paged path splits 64-row host Q tiles into HB blocks along grid.x (host maps unchanged)

Benchmark: NVIDIA RTX 6000D, short-q/long-kv prefill 1.4-3.4x (G=8 B=16 q=16 kv=16k 4.22 -> 1.26 ms); full prefill/MHA/paged unchanged (compute-bound); verified vs SDPA G in {1,2,3,4,8,32}, 99 tests pass
This commit is contained in:
2026-08-31 13:39:47 +08:00
parent 7dd184a4e5
commit a1a1a6bf0f
5 changed files with 91 additions and 20 deletions
@@ -56,6 +56,20 @@ struct DenseQSchedule {
q_tile = blockIdx.x;
}
// GQA-packed prefill mapping: HB q-heads of one kv-head group share a
// block's K/V stream, each head owning `rows` = BR*WPH consecutive q rows
// per block. Dense tensors tile q_len directly, one block per range.
HOST_FORCEINLINE int packed_grid_x(
const AttentionParams<bf16>& p, int rows) {
return (p.q_len + rows - 1) / rows;
}
DEVICE_FORCEINLINE void map_packed_block(
const AttentionParams<bf16>&, int rows, int& batch, int& row_base) {
batch = blockIdx.z;
row_base = blockIdx.x * rows;
}
DEVICE_FORCEINLINE int q_len(
const AttentionParams<bf16>& p, int) {
return p.q_len;
@@ -84,6 +98,23 @@ struct PackedQSchedule {
q_tile = p.q_tile_to_index[blockIdx.x];
}
// GQA-packed prefill mapping: the host tile maps are built in
// HOST_Q_TILE_ROWS granularity, so each host tile splits into
// HOST_Q_TILE_ROWS / rows packed blocks along blockIdx.x.
HOST_FORCEINLINE int packed_grid_x(
const AttentionParams<bf16>& p, int rows) {
return p.num_q_tiles * (HOST_Q_TILE_ROWS / rows);
}
DEVICE_FORCEINLINE void map_packed_block(
const AttentionParams<bf16>& p, int rows, int& batch, int& row_base) {
const int hb = HOST_Q_TILE_ROWS / rows;
const int host_tile = blockIdx.x / hb;
batch = p.q_tile_to_batch[host_tile];
row_base = p.q_tile_to_index[host_tile] * HOST_Q_TILE_ROWS
+ (blockIdx.x - host_tile * hb) * rows;
}
DEVICE_FORCEINLINE int q_len(
const AttentionParams<bf16>& p, int batch) {
return p.qo_indptr[batch + 1] - p.qo_indptr[batch];