fix: synchronize final decode async copy

- wait for the final split-KV tile before reading shared memory
- cover long decode with production context capacity
This commit is contained in:
2026-08-09 00:31:47 +08:00
parent 7f0e8bb8c2
commit a33ca04f60
2 changed files with 11 additions and 4 deletions
+4 -1
View File
@@ -121,7 +121,10 @@ __global__ void attn_decode_split_kv_mma_kernel(AttentionParams<bf16> p) {
load_tile(ti_begin + i, i);
for (int it = 0; it < ntiles; it++) {
cp_async_wait_group<STAGES - 1>();
if (it + 1 == ntiles)
cp_async_wait_group<0>();
else
cp_async_wait_group<STAGES - 1>();
__syncwarp();
process_tile(it, it & (STAGES - 1));
__syncwarp();
+7 -3
View File
@@ -127,14 +127,15 @@ inline void print_paged_row(const char* cfg, float max_err, bool pass) {
// ======================================================================
template <int HEAD_DIM>
static int run_decode_test(int B, int Hq, int Hkv, int max_seq,
int causal, int seed) {
int causal, int seed, int context_capacity = 0,
int fixed_seq_len = 0) {
// Variable seq_lens per request
srand(seed);
std::vector<int> seq_lens(B);
for (int b = 0; b < B; b++)
seq_lens[b] = 8 + rand() % (max_seq - 8);
seq_lens[b] = fixed_seq_len ? fixed_seq_len : 8 + rand() % (max_seq - 8);
int max_sl = *std::max_element(seq_lens.begin(), seq_lens.end());
int max_ctx = max_sl + 16;
int max_ctx = context_capacity ? context_capacity : max_sl + 16;
int pool_size = B * max_ctx;
int num_reqs = B + 4;
@@ -844,6 +845,9 @@ int main() {
fail += run_decode_test<256>(1, 2, 1, 256, 0, 9);
fail += run_decode_test<128>(16, 32, 4, 2048, 0, 10);
fail += run_decode_test<128>(32, 32, 4, 1024, 0, 11);
// Production keeps a fixed 32768-wide request table. This forces 32
// splits, so seq_len > 512 gives each split multiple cp.async tiles.
fail += run_decode_test<64>(1, 24, 4, 1100, 0, 12, 32768, 1100);
// Decode with 2D mask (regression: mixed seq_lens + HasMask)
fail += run_decode_mask_test<128>(2, 8, 2, 256, 30);