0z5a
1c3515714f
- Replace per-element loads with 128-bit uint4 vectorized loads (8 halves per access), improving every measured shape: q/k/v at M=2 from 6.0us to 5.4us, q_proj speedup 2.28-2.45x, mlp_down at M=4 2.76x, lm_head at M=1 +6-8%
- Extend kernel M support from {1,2,4,8} to all M in 1-8 via new BLOCK_M cases 3,5,6,7, since cuBLAS wmma templates pad small M to 8/16 rows and waste compute
- Keep the auto-dispatch allowlist unchanged: a 64-step greedy-walk probe on the real decode path showed mlp_down (K=6912) divergence at step 1 and argmax flips for every candidate odd-M band, the same noise class already present in the merged M=2/4 entries, so no entry has the stability evidence the gate requires
- Rejected alternatives with measurements: split-K accumulation (k/v shapes regress 6.0us to 9.2us, code removed) and MMA tiles (small M is DRAM-bound at ~1 FLOP/byte vs the ~138 needed)
- Update test_gemv M-rejection case to M=9 and test_linear_dispatch multirow fallback to M=9 for the widened range
Benchmark: 8x L20 (sm_89, CUDA 12.8), single-GPU microbench, 200 iters after 20 warmup, weights L2-resident; q(1536x1536) M=3 8.9->5.3us, kv(256x1536) M=3 8.7->3.0us, down(1536x6912) M=3 53.5->10.3us; full gate 691 passed, test_bf16_gemv_uses_current_stream passes in isolation after GPU contention rerun
156 lines
5.0 KiB
Python
156 lines
5.0 KiB
Python
import pytest
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
from astrai.extension import bf16_gemv, is_available
|
|
|
|
GEMV_AVAILABLE = (
|
|
torch.cuda.is_available()
|
|
and is_available("bf16_gemv")
|
|
and torch.cuda.get_device_capability() >= (8, 0)
|
|
)
|
|
skip_no_gemv = pytest.mark.skipif(
|
|
not GEMV_AVAILABLE,
|
|
reason="BF16 GEMV requires a built kernel and compute capability 8.0+",
|
|
)
|
|
|
|
|
|
@skip_no_gemv
|
|
@pytest.mark.parametrize(
|
|
"n,k",
|
|
[(256, 1536), (1536, 1536), (6912, 1536), (1536, 6912), (100000, 1536)],
|
|
)
|
|
def test_bf16_gemv_matches_linear_shape_families(n, k):
|
|
torch.manual_seed(17)
|
|
x = torch.randn(k, device="cuda", dtype=torch.bfloat16)
|
|
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16)
|
|
actual = bf16_gemv(x, weight)
|
|
expected = F.linear(x, weight)
|
|
assert actual.shape == (n,)
|
|
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
|
|
|
|
|
|
@skip_no_gemv
|
|
@pytest.mark.parametrize("m", [2, 3, 4, 5, 6, 7, 8])
|
|
@pytest.mark.parametrize("n,k", [(256, 1536), (1536, 1536), (1536, 6912)])
|
|
def test_bf16_gemv_matches_small_decode_batches(m, n, k):
|
|
torch.manual_seed(19 + m)
|
|
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
|
|
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16)
|
|
actual = bf16_gemv(x, weight)
|
|
expected = F.linear(x, weight)
|
|
assert actual.shape == (m, n)
|
|
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.5)
|
|
|
|
|
|
@skip_no_gemv
|
|
def test_bf16_gemv_preserves_singleton_batch_and_fuses_bias():
|
|
torch.manual_seed(23)
|
|
x = torch.randn(1, 1536, device="cuda", dtype=torch.bfloat16)
|
|
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
|
|
bias = torch.randn(1536, device="cuda", dtype=torch.bfloat16)
|
|
actual = bf16_gemv(x, weight, bias)
|
|
expected = F.linear(x, weight, bias)
|
|
assert actual.shape == (1, 1536)
|
|
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
|
|
|
|
|
|
@skip_no_gemv
|
|
def test_bf16_gemv_small_batch_fuses_bias():
|
|
torch.manual_seed(25)
|
|
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16)
|
|
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
|
|
bias = torch.randn(256, device="cuda", dtype=torch.bfloat16)
|
|
actual = bf16_gemv(x, weight, bias)
|
|
expected = F.linear(x, weight, bias)
|
|
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
|
|
|
|
|
|
@skip_no_gemv
|
|
def test_bf16_gemv_uses_current_stream():
|
|
x = torch.randn(1536, device="cuda", dtype=torch.bfloat16)
|
|
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
|
|
stream = torch.cuda.Stream()
|
|
with torch.cuda.stream(stream):
|
|
actual = bf16_gemv(x, weight)
|
|
expected = F.linear(x, weight)
|
|
stream.synchronize()
|
|
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
|
|
|
|
|
|
@skip_no_gemv
|
|
def test_bf16_gemv_cuda_graph_replay():
|
|
torch.manual_seed(29)
|
|
x = torch.randn(1536, device="cuda", dtype=torch.bfloat16)
|
|
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
|
|
for _ in range(3):
|
|
bf16_gemv(x, weight)
|
|
|
|
graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(graph):
|
|
actual = bf16_gemv(x, weight)
|
|
|
|
x.copy_(torch.randn_like(x))
|
|
graph.replay()
|
|
expected = F.linear(x, weight)
|
|
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
|
|
|
|
|
|
@skip_no_gemv
|
|
def test_bf16_gemv_small_batch_cuda_graph_replay():
|
|
torch.manual_seed(31)
|
|
x = torch.randn(8, 1536, device="cuda", dtype=torch.bfloat16)
|
|
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
|
|
for _ in range(3):
|
|
bf16_gemv(x, weight)
|
|
|
|
graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(graph):
|
|
actual = bf16_gemv(x, weight)
|
|
|
|
x.copy_(torch.randn_like(x))
|
|
graph.replay()
|
|
expected = F.linear(x, weight)
|
|
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
|
|
|
|
|
|
@skip_no_gemv
|
|
@pytest.mark.parametrize(
|
|
"make_args,error",
|
|
[
|
|
(
|
|
lambda: (
|
|
torch.randn(9, 16, device="cuda", dtype=torch.bfloat16),
|
|
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
|
|
),
|
|
"M must",
|
|
),
|
|
(
|
|
lambda: (
|
|
torch.randn(15, device="cuda", dtype=torch.bfloat16),
|
|
torch.randn(8, 15, device="cuda", dtype=torch.bfloat16),
|
|
),
|
|
"even",
|
|
),
|
|
(
|
|
lambda: (
|
|
torch.randn(16, device="cuda", dtype=torch.float16),
|
|
torch.randn(8, 16, device="cuda", dtype=torch.float16),
|
|
),
|
|
"bf16",
|
|
),
|
|
(
|
|
lambda: (
|
|
torch.randn(
|
|
16, device="cuda", dtype=torch.bfloat16, requires_grad=True
|
|
),
|
|
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
|
|
),
|
|
"autograd",
|
|
),
|
|
],
|
|
)
|
|
def test_bf16_gemv_rejects_unsupported_inputs(make_args, error):
|
|
with pytest.raises(RuntimeError, match=error):
|
|
bf16_gemv(*make_args())
|