perf: tune bf16 gemv and add opt-in fused swiglu

- deepen common-shape BF16 GEMV tuning with warp-row tiling for LLaMA/Qwen2/GPT-NeoX/OPT decode projections - add fused BF16 up/gate SwiGLU CUDA primitive with ASTRAI_SWIGLU=0/1/auto dispatch - keep the unfused linear backend as the default path; auto enables no shape until per-architecture checkpoint gates pass - fall back to the linear/torch chain when kernels are absent, on CPU, in training, or outside supported M/K/dtype shapes - add gemv/swiglu benchmark scripts, dispatch and parity tests, and kernel documentation Benchmark: NVIDIA L20 (sm_89), CUDA 12.8, PyTorch 2.11.0+cu128, idle GPU. AstrAI 1B config (24 layers, hidden 1536, vocab 100000), BF16, prompt 128, 32 greedy decode tokens, CUDA graphs enabled, A/B in separate interleaved processes (3 rounds, 8 trials each, medians). Default vs ASTRAI_SWIGLU=1 per generate call: batch 1 134.8->129.1 ms (+4.44%), batch 2 136.2->130.9 ms (+4.06%), batch 4 145.5->140.3 ms (+3.66%). Greedy output identical at batch 1, differs at batch 2/4, so auto stays unfused by default; kernelless fallback verified bit-identical greedy.
2026-09-03 04:26:53 +08:00
parent 88c06db096
commit d4a292b36b
20 changed files with 2008 additions and 37 deletions
@@ -0,0 +1,99 @@
+import pytest
+import torch
+import torch.nn.functional as F
+
+from astrai.extension import bf16_swiglu, is_available
+
+SWIGLU_AVAILABLE = (
+    torch.cuda.is_available()
+    and is_available("bf16_swiglu")
+    and torch.cuda.get_device_capability() >= (8, 0)
+)
+skip_no_swiglu = pytest.mark.skipif(
+    not SWIGLU_AVAILABLE,
+    reason="BF16 SwiGLU requires a built kernel and compute capability 8.0+",
+)
+
+
+def reference_swiglu(x, up_weight, gate_weight):
+    return F.linear(x, up_weight) * F.silu(F.linear(x, gate_weight))
+
+
+@skip_no_swiglu
+@pytest.mark.parametrize("m", [1, 2, 4, 8])
+@pytest.mark.parametrize("n,k", [(6912, 1536), (4096, 4096), (11008, 4096)])
+def test_bf16_swiglu_matches_common_dense_mlp_shapes(m, n, k):
+    torch.manual_seed(37 + m)
+    x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1
+    up_weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
+    gate_weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
+    actual = bf16_swiglu(x, up_weight, gate_weight)
+    expected = reference_swiglu(x, up_weight, gate_weight)
+    assert actual.shape == (m, n)
+    torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
+
+
+@skip_no_swiglu
+def test_bf16_swiglu_preserves_vector_shape():
+    x = torch.randn(1536, device="cuda", dtype=torch.bfloat16) * 0.1
+    up_weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
+    gate_weight = torch.randn_like(up_weight) * 0.02
+    actual = bf16_swiglu(x, up_weight, gate_weight)
+    expected = reference_swiglu(x, up_weight, gate_weight)
+    assert actual.shape == (256,)
+    torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
+
+
+@skip_no_swiglu
+def test_bf16_swiglu_uses_current_stream_and_cuda_graph():
+    torch.manual_seed(43)
+    x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
+    up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
+    gate_weight = torch.randn_like(up_weight) * 0.02
+    stream = torch.cuda.Stream()
+    with torch.cuda.stream(stream):
+        for _ in range(3):
+            bf16_swiglu(x, up_weight, gate_weight)
+        graph = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(graph):
+            actual = bf16_swiglu(x, up_weight, gate_weight)
+        x.copy_(torch.randn_like(x) * 0.1)
+        graph.replay()
+    stream.synchronize()
+    expected = reference_swiglu(x, up_weight, gate_weight)
+    torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
+
+
+@skip_no_swiglu
+@pytest.mark.parametrize(
+    "make_args,error",
+    [
+        (
+            lambda: (
+                torch.randn(9, 16, device="cuda", dtype=torch.bfloat16),
+                torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
+                torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
+            ),
+            "M must",
+        ),
+        (
+            lambda: (
+                torch.randn(2, 15, device="cuda", dtype=torch.bfloat16),
+                torch.randn(8, 15, device="cuda", dtype=torch.bfloat16),
+                torch.randn(8, 15, device="cuda", dtype=torch.bfloat16),
+            ),
+            "divisible by 8",
+        ),
+        (
+            lambda: (
+                torch.randn(2, 16, device="cuda", dtype=torch.bfloat16),
+                torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
+                torch.randn(7, 16, device="cuda", dtype=torch.bfloat16),
+            ),
+            "identical shapes",
+        ),
+    ],
+)
+def test_bf16_swiglu_rejects_unsupported_inputs(make_args, error):
+    with pytest.raises(RuntimeError, match=error):
+        bf16_swiglu(*make_args())