ViperEkura/AstrAI
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf: tune bf16 gemv and add opt-in fused swiglu

Browse Source 
- deepen common-shape BF16 GEMV tuning with warp-row tiling for LLaMA/Qwen2/GPT-NeoX/OPT decode projections
- add fused BF16 up/gate SwiGLU CUDA primitive with ASTRAI_SWIGLU=0/1/auto dispatch
- keep the unfused linear backend as the default path; auto enables no shape until per-architecture checkpoint gates pass
- fall back to the linear/torch chain when kernels are absent, on CPU, in training, or outside supported M/K/dtype shapes
- add gemv/swiglu benchmark scripts, dispatch and parity tests, and kernel documentation

Benchmark: NVIDIA L20 (sm_89), CUDA 12.8, PyTorch 2.11.0+cu128, idle GPU. AstrAI 1B config (24 layers, hidden 1536, vocab 100000), BF16, prompt 128, 32 greedy decode tokens, CUDA graphs enabled, A/B in separate interleaved processes (3 rounds, 8 trials each, medians). Default vs ASTRAI_SWIGLU=1 per generate call: batch 1 134.8->129.1 ms (+4.44%), batch 2 136.2->130.9 ms (+4.06%), batch 4 145.5->140.3 ms (+3.66%). Greedy output identical at batch 1, differs at batch 2/4, so auto stays unfused by default; kernelless fallback verified bit-identical greedy.
This commit is contained in:
0z5a
2026-09-03 04:26:53 +08:00
parent 88c06db096
commit d4a292b36b
20 changed files with 2008 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/extension/test_gemv.py
+60
View File
@@ -43,6 +43,66 @@ def test_bf16_gemv_matches_small_decode_batches(m, n, k):
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.5)
@skip_no_gemv
@pytest.mark.parametrize("m", [2, 4])
@pytest.mark.parametrize(
"n,k",
[
(1024, 4096),
(4096, 4096),
(11008, 4096),
(4096, 11008),
(14336, 4096),
(4096, 14336),
(5120, 5120),
(13824, 5120),
(5120, 13824),
(16384, 4096),
(4096, 16384),
(512, 3584),
(3584, 3584),
(18944, 3584),
(3584, 18944),
(1024, 8192),
(8192, 8192),
(28672, 8192),
(8192, 28672),
(2048, 2048),
(8192, 2048),
(2048, 8192),
],
)
def test_bf16_gemv_matches_common_transformer_shapes(m, n, k):
torch.manual_seed(2026 + m + n + k)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.empty(n, k, device="cuda", dtype=torch.bfloat16)
weight.normal_(mean=0.0, std=0.02)
actual = bf16_gemv(x, weight)
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemv
@pytest.mark.parametrize(
"m,n,k",
[
(1, 8192, 2048),
(8, 4096, 11008),
(8, 512, 3584),
(8, 1024, 8192),
(8, 2048, 8192),
],
)
def test_bf16_gemv_matches_half_cta_edge_bands(m, n, k):
torch.manual_seed(2026 + m + n + k)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.empty(n, k, device="cuda", dtype=torch.bfloat16)
weight.normal_(mean=0.0, std=0.02)
actual = bf16_gemv(x, weight)
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemv
def test_bf16_gemv_preserves_singleton_batch_and_fuses_bias():
torch.manual_seed(23)
tests/extension/test_linear_dispatch.py
+84 -4
View File
@@ -6,6 +6,7 @@ import torch.nn.functional as F
from astrai.extension import explain, is_available, linear, op_backend
from astrai.extension.backend import linear as public_linear
from astrai.extension.backend.linear import _AUTO_GEMV_SHAPES
from astrai.model.components.linear import Linear
GEMV_AVAILABLE = (
@@ -23,6 +24,45 @@ def test_linear_backend_is_public():
assert linear is public_linear
def test_sm89_common_shape_policy_keeps_only_validated_families_enabled():
common = {
(1024, 4096),
(4096, 4096),
(11008, 4096),
(4096, 11008),
(14336, 4096),
(4096, 14336),
(5120, 5120),
(13824, 5120),
(5120, 13824),
(16384, 4096),
(4096, 16384),
}
subthreshold_m4 = {(4096, 4096), (11008, 4096), (4096, 11008)}
qwen2_7b = {
(512, 3584),
(3584, 3584),
(18944, 3584),
(3584, 18944),
}
llama3_70b = {
(1024, 8192),
(8192, 8192),
(28672, 8192),
(8192, 28672),
}
opt_1_3b = {(2048, 2048), (8192, 2048), (2048, 8192)}
policy = _AUTO_GEMV_SHAPES[(8, 9)]
assert policy[1] == opt_1_3b
assert common <= policy[2]
assert qwen2_7b | llama3_70b | opt_1_3b <= policy[2]
assert common - subthreshold_m4 <= policy[4]
assert qwen2_7b | llama3_70b <= policy[4]
assert subthreshold_m4.isdisjoint(policy[4])
assert opt_1_3b.isdisjoint(policy[4])
assert 8 not in policy
def test_model_linear_routes_through_backend(monkeypatch):
sentinel = torch.randn(2, 4)
@@ -93,7 +133,7 @@ def test_mode_one_dispatches_supported_small_batches(monkeypatch, m):
@skip_no_gemv
def test_auto_m1_falls_back_until_end_to_end_gate_passes(monkeypatch):
def test_auto_unmeasured_m1_falls_back(monkeypatch):
monkeypatch.setenv("ASTRAI_GEMV", "auto")
x = torch.randn(1, 1536, device="cuda", dtype=torch.bfloat16)
winning = torch.randn(
@@ -109,10 +149,38 @@ def test_auto_m1_falls_back_until_end_to_end_gate_passes(monkeypatch):
@skip_no_gemv
def test_auto_selects_measured_sm89_small_batch_winner(monkeypatch):
@pytest.mark.parametrize(
"m,n,k",
[
(4, 256, 1536),
(2, 1024, 4096),
(2, 11008, 4096),
(2, 4096, 11008),
(2, 14336, 4096),
(4, 4096, 14336),
(2, 5120, 5120),
(4, 13824, 5120),
(2, 5120, 13824),
(4, 16384, 4096),
(2, 4096, 16384),
(2, 512, 3584),
(4, 3584, 3584),
(2, 18944, 3584),
(4, 3584, 18944),
(2, 1024, 8192),
(4, 8192, 8192),
(2, 28672, 8192),
(4, 8192, 28672),
(1, 2048, 2048),
(2, 8192, 2048),
(1, 2048, 8192),
],
)
def test_auto_selects_measured_sm89_small_batch_winner(monkeypatch, m, n, k):
monkeypatch.setenv("ASTRAI_GEMV", "auto")
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.empty(n, k, device="cuda", dtype=torch.bfloat16)
weight.normal_(mean=0.0, std=0.02)
with torch.no_grad():
trace = explain("linear", x, weight)
if torch.cuda.get_device_capability() == (8, 9):
@@ -133,6 +201,18 @@ def test_auto_selects_measured_sm89_small_batch_winner(monkeypatch):
(4, 100000, 1536), # LM head misses the 5% M=4 gate
(4, 1536, 6912), # long-K accumulation changed checkpoint greedy output
(8, 256, 1536), # remaining M=8 winners miss the 3% end-to-end gate
(1, 4096, 4096), # isolated M=1 winner misses the projection-chain gate
(8, 1024, 4096), # isolated M=8 winner misses the projection-chain gate
(4, 12288, 4096), # GPT-NeoX fused QKV was not measured as a winner
(4, 4096, 4096), # LLaMA 2 7B M=4 chain misses the 3% gate
(4, 11008, 4096),
(4, 4096, 11008),
(1, 3584, 3584), # Qwen2 M=1 chain misses the 3% gate
(8, 3584, 3584), # Qwen2 Q/O misses the M=8 per-shape gate
(1, 8192, 8192), # LLaMA 3 70B M=1 projections miss the per-shape gate
(8, 1024, 8192), # LLaMA 3 70B K/V loses at wrapper level for M=8
(4, 8192, 2048), # OPT up loses at wrapper level for M=4
(8, 2048, 2048), # OPT M=8 chain and Q/K/V/O both regress
],
)
def test_auto_rejects_measured_small_batch_losers(monkeypatch, m, n, k):
tests/extension/test_swiglu.py
+99
View File
@@ -0,0 +1,99 @@
import pytest
import torch
import torch.nn.functional as F
from astrai.extension import bf16_swiglu, is_available
SWIGLU_AVAILABLE = (
torch.cuda.is_available()
and is_available("bf16_swiglu")
and torch.cuda.get_device_capability() >= (8, 0)
)
skip_no_swiglu = pytest.mark.skipif(
not SWIGLU_AVAILABLE,
reason="BF16 SwiGLU requires a built kernel and compute capability 8.0+",
)
def reference_swiglu(x, up_weight, gate_weight):
return F.linear(x, up_weight) * F.silu(F.linear(x, gate_weight))
@skip_no_swiglu
@pytest.mark.parametrize("m", [1, 2, 4, 8])
@pytest.mark.parametrize("n,k", [(6912, 1536), (4096, 4096), (11008, 4096)])
def test_bf16_swiglu_matches_common_dense_mlp_shapes(m, n, k):
torch.manual_seed(37 + m)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
gate_weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
actual = bf16_swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
assert actual.shape == (m, n)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
@skip_no_swiglu
def test_bf16_swiglu_preserves_vector_shape():
x = torch.randn(1536, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight) * 0.02
actual = bf16_swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
assert actual.shape == (256,)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
@skip_no_swiglu
def test_bf16_swiglu_uses_current_stream_and_cuda_graph():
torch.manual_seed(43)
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight) * 0.02
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
for _ in range(3):
bf16_swiglu(x, up_weight, gate_weight)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = bf16_swiglu(x, up_weight, gate_weight)
x.copy_(torch.randn_like(x) * 0.1)
graph.replay()
stream.synchronize()
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
@skip_no_swiglu
@pytest.mark.parametrize(
"make_args,error",
[
(
lambda: (
torch.randn(9, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
),
"M must",
),
(
lambda: (
torch.randn(2, 15, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 15, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 15, device="cuda", dtype=torch.bfloat16),
),
"divisible by 8",
),
(
lambda: (
torch.randn(2, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(7, 16, device="cuda", dtype=torch.bfloat16),
),
"identical shapes",
),
],
)
def test_bf16_swiglu_rejects_unsupported_inputs(make_args, error):
with pytest.raises(RuntimeError, match=error):
bf16_swiglu(*make_args())
tests/extension/test_swiglu_dispatch.py
+94
View File
@@ -0,0 +1,94 @@
import logging
import pytest
import torch
import torch.nn.functional as F
from astrai.extension import is_available, swiglu
from astrai.model.components.mlp import MLP
SWIGLU_AVAILABLE = (
torch.cuda.is_available()
and is_available("bf16_swiglu")
and torch.cuda.get_device_capability() >= (8, 0)
)
skip_no_swiglu = pytest.mark.skipif(
not SWIGLU_AVAILABLE,
reason="BF16 SwiGLU requires a built kernel and compute capability 8.0+",
)
def reference_swiglu(x, up_weight, gate_weight):
return F.linear(x, up_weight) * F.silu(F.linear(x, gate_weight))
def test_cpu_and_training_calls_fall_back_with_gradients(monkeypatch):
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
x = torch.randn(2, 8, requires_grad=True)
up_weight = torch.randn(4, 8, requires_grad=True)
gate_weight = torch.randn(4, 8, requires_grad=True)
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected)
actual.sum().backward()
assert x.grad is not None
assert up_weight.grad is not None
assert gate_weight.grad is not None
def test_invalid_mode_warns_and_uses_auto(monkeypatch, caplog):
monkeypatch.setenv("ASTRAI_SWIGLU", "invalid-test-mode")
with caplog.at_level(logging.WARNING):
actual = swiglu(torch.randn(2, 8), torch.randn(4, 8), torch.randn(4, 8))
assert actual.shape == (2, 4)
assert "using auto" in caplog.text
def test_mlp_routes_through_swiglu_backend(monkeypatch):
sentinel = torch.randn(2, 4)
def fake_swiglu(x, up_weight, gate_weight):
assert x.shape == (2, 3)
assert up_weight.shape == gate_weight.shape == (4, 3)
return sentinel
monkeypatch.setattr("astrai.model.components.mlp.swiglu", fake_swiglu)
layer = MLP(3, 4)
output = layer(torch.randn(2, 3))
assert output["hidden_states"].shape == (2, 3)
@skip_no_swiglu
def test_mode_zero_disables_fused_kernel(monkeypatch):
monkeypatch.setenv("ASTRAI_SWIGLU", "0")
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight) * 0.02
with torch.no_grad():
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected)
@skip_no_swiglu
def test_mode_one_forces_supported_shape(monkeypatch):
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight) * 0.02
with torch.no_grad():
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
@skip_no_swiglu
def test_auto_falls_back_until_shape_is_qualified(monkeypatch):
monkeypatch.setenv("ASTRAI_SWIGLU", "auto")
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16)
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16)
gate_weight = torch.randn_like(up_weight)
with torch.no_grad():
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected)