ViperEkura
1798474316
- split-K removed entirely: tiled kernel walks K in one pass, no partials/semas workspace, no memset, single launch per call
- skinny GEMM (M<=8) dispatch table replaces the hand-written switch
- shape-driven four-family table replaces plan_gemm: wide-N (n>=4096) default {16,64,64,3,128} with BM=32 at M>16; narrow-N deep-K rings {16,32,256,2,64} while the grid fits one wave, {16,32,128,2,64} past it
- narrow-N is K-serial: widening the grid measurably does nothing (BN 64->32 ties, doubled m_tiles tie, kv at 4 blocks ties q/o at 24); deeper K chunks win until 72KB smem forces one CTA per SM and past one wave the 2-wave quantization loses to BK=128
- launch-check macros in common/launch.cuh; smem opt-in for the 72KB/60KB rings
- rename kernels/bf16_*.cu to gemm.cu/swiglu.cu; module names unchanged
- Python gate: lm_head (N>32768) falls back to cuBLAS, band narrows to M<=32
- drop the stale per-op benchmark narratives; fold the live numbers into cuda_kernels.md
Benchmark: NVIDIA L20 (sm_89, 92 SMs), CUDA 12.8, bf16, L2-thrash weight rotation, per-call medians at M=16: q/o 9.5us, kv 8.6us, gate/up 33.3us, down 33.7us (down -29% vs prior default). End-to-end 1B decode (gen 128, 3 trials, tokens/s vs cuBLAS): B=1 260 vs 252, B=8 1660 vs 1446, B=16 2464 vs 2437, B=32 3620 vs 3690. Prior split-K dispatch measured B=16 2243 / B=32 3393.
119 lines
4.6 KiB
Python
119 lines
4.6 KiB
Python
import logging
|
|
|
|
import pytest
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
from astrai.extension import is_available, swiglu
|
|
from astrai.model.components.mlp import MLP
|
|
|
|
SWIGLU_AVAILABLE = (
|
|
torch.cuda.is_available()
|
|
and is_available("bf16_swiglu")
|
|
and torch.cuda.get_device_capability() >= (8, 0)
|
|
)
|
|
skip_no_swiglu = pytest.mark.skipif(
|
|
not SWIGLU_AVAILABLE,
|
|
reason="BF16 SwiGLU requires a built kernel and compute capability 8.0+",
|
|
)
|
|
|
|
|
|
def reference_swiglu(x, up_weight, gate_weight):
|
|
return F.linear(x, up_weight) * F.silu(F.linear(x, gate_weight))
|
|
|
|
|
|
def test_cpu_and_training_calls_fall_back_with_gradients(monkeypatch):
|
|
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
|
|
x = torch.randn(2, 8, requires_grad=True)
|
|
up_weight = torch.randn(4, 8, requires_grad=True)
|
|
gate_weight = torch.randn(4, 8, requires_grad=True)
|
|
actual = swiglu(x, up_weight, gate_weight)
|
|
expected = reference_swiglu(x, up_weight, gate_weight)
|
|
torch.testing.assert_close(actual, expected)
|
|
actual.sum().backward()
|
|
assert x.grad is not None
|
|
assert up_weight.grad is not None
|
|
assert gate_weight.grad is not None
|
|
|
|
|
|
def test_invalid_mode_warns_and_uses_auto(monkeypatch, caplog):
|
|
monkeypatch.setenv("ASTRAI_SWIGLU", "invalid-test-mode")
|
|
x = torch.randn(2, 8)
|
|
up_weight = torch.randn(4, 8)
|
|
gate_weight = torch.randn(4, 8)
|
|
with caplog.at_level(logging.WARNING):
|
|
actual = swiglu(x, up_weight, gate_weight)
|
|
assert "using auto" in caplog.text
|
|
torch.testing.assert_close(actual, reference_swiglu(x, up_weight, gate_weight))
|
|
|
|
|
|
def test_mlp_routes_through_swiglu_backend(monkeypatch):
|
|
sentinel = torch.randn(2, 4)
|
|
|
|
def fake_swiglu(x, up_weight, gate_weight):
|
|
assert x.shape == (2, 3)
|
|
assert up_weight.shape == gate_weight.shape == (4, 3)
|
|
return sentinel
|
|
|
|
monkeypatch.setattr("astrai.model.components.mlp.swiglu", fake_swiglu)
|
|
layer = MLP(3, 4)
|
|
output = layer(torch.randn(2, 3))
|
|
assert output["hidden_states"].shape == (2, 3)
|
|
|
|
|
|
@skip_no_swiglu
|
|
def test_mode_zero_disables_fused_kernel(monkeypatch):
|
|
monkeypatch.setenv("ASTRAI_SWIGLU", "0")
|
|
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
|
|
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
|
|
gate_weight = torch.randn_like(up_weight) * 0.02
|
|
with torch.no_grad():
|
|
actual = swiglu(x, up_weight, gate_weight)
|
|
expected = reference_swiglu(x, up_weight, gate_weight)
|
|
torch.testing.assert_close(actual, expected)
|
|
|
|
|
|
@skip_no_swiglu
|
|
def test_mode_one_forces_supported_shape(monkeypatch):
|
|
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
|
|
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
|
|
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
|
|
gate_weight = torch.randn_like(up_weight) * 0.02
|
|
with torch.no_grad():
|
|
actual = swiglu(x, up_weight, gate_weight)
|
|
expected = reference_swiglu(x, up_weight, gate_weight)
|
|
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
|
|
|
|
|
|
@skip_no_swiglu
|
|
def test_auto_uses_fused_chain_for_decode_batches(monkeypatch):
|
|
# Auto adopts the fused primitive for the decode band; numerics match
|
|
# the unfused linear-backend chain within BF16 accumulation-order noise.
|
|
monkeypatch.setenv("ASTRAI_SWIGLU", "auto")
|
|
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16)
|
|
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16)
|
|
gate_weight = torch.randn_like(up_weight)
|
|
with torch.no_grad():
|
|
actual = swiglu(x, up_weight, gate_weight)
|
|
expected = reference_swiglu(x, up_weight, gate_weight)
|
|
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.1)
|
|
|
|
|
|
@skip_no_swiglu
|
|
def test_mode_one_falls_back_for_misaligned_storage(monkeypatch):
|
|
"""Contiguous-but-offset views must route to the unfused torch chain
|
|
even with ASTRAI_SWIGLU=1 instead of reaching the uint4-only kernel
|
|
(regression: the fused primitive faulted with a misaligned-address
|
|
CUDA error for such inputs)."""
|
|
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
|
|
k = 1536
|
|
x_base = torch.randn(2 * k + 8, device="cuda", dtype=torch.bfloat16) * 0.1
|
|
x = x_base[1 : 1 + 2 * k].view(2, k)
|
|
assert x.is_contiguous() and (x.data_ptr() & 15) != 0
|
|
up_weight = torch.randn(64, k, device="cuda", dtype=torch.bfloat16) * 0.02
|
|
gate_weight = torch.randn_like(up_weight)
|
|
with torch.no_grad():
|
|
actual = swiglu(x, up_weight, gate_weight)
|
|
expected = reference_swiglu(x, up_weight, gate_weight)
|
|
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
|