ViperEkura/AstrAI
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: remove bf16 gemm and swiglu kernels and rebuild csrc benchmarks

Browse Source 
- delete csrc/kernels/gemm.cu and swiglu.cu and drop their CMake and setup.py registration
- remove the ops wrappers plus backend/linear.py and backend/swiglu.py so Linear and MLP call F.linear directly
- drop the four gemm and swiglu kernel test files and prune the stale cuda_kernels.md sections
- add csrc/bench benchmarks for the remaining kernels: attention decode prefill paged decode paged prefill versus single-launch SDPA references, rotary versus the torch fallback, fp8 quantize and mm_fp8 versus torch baselines
- attention, rotary_emb, and fp8_ops kernels are unchanged
This commit is contained in:
ViperEkura
2026-09-05 01:38:10 +08:00
parent a77e35dd51
commit 6709534d64
24 changed files with 1306 additions and 3394 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/extension/test_gemm.py
-396
View File
@@ -1,396 +0,0 @@
import pytest
import torch
import torch.nn.functional as F
from astrai.extension import bf16_gemm, is_available
GEMM_AVAILABLE = (
torch.cuda.is_available()
and is_available("bf16_gemm")
and torch.cuda.get_device_capability() >= (8, 0)
)
skip_no_gemm = pytest.mark.skipif(
not GEMM_AVAILABLE,
reason="BF16 GEMM requires a built kernel and compute capability 8.0+",
)
def _assert_close_fp64(actual, x, weight, bias=None):
"""Compare bf16 kernel output vs fp64-exact with ulp-scaled tolerance.
Avoids false failures from cuBLAS default bf16 split-K partial reduction
(which can introduce ~2 ulp diffs on near-tie rounding at long K). Used
for tiled-path tests (M > 8, K >= 4096) where the bf16 accumulation tie
pattern may differ from cuBLAS's."""
exact = x.double() @ weight.double().T
if bias is not None:
exact = exact + bias.double()
ulp = (exact.abs() * 2**-9).clamp(min=2**-9)
max_ulp = ((actual.double() - exact).abs() / ulp).max().item()
assert max_ulp < 8, (
f"max_ulp={max_ulp:.1f} exceeds 8 (exact fp32 accumulation should stay within ~2 ulps)"
)
@skip_no_gemm
@pytest.mark.parametrize(
"n,k",
[(256, 1536), (1536, 1536), (6912, 1536), (1536, 6912), (100000, 1536)],
)
def test_bf16_gemm_matches_linear_shape_families(n, k):
torch.manual_seed(17)
x = torch.randn(k, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16)
actual = bf16_gemm(x, weight)
expected = F.linear(x, weight)
assert actual.shape == (n,)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemm
@pytest.mark.parametrize("m", [2, 3, 4, 5, 6, 7, 8])
@pytest.mark.parametrize("n,k", [(256, 1536), (1536, 1536), (1536, 6912)])
def test_bf16_gemm_matches_small_decode_batches(m, n, k):
torch.manual_seed(19 + m)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16)
actual = bf16_gemm(x, weight)
expected = F.linear(x, weight)
assert actual.shape == (m, n)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.5)
@skip_no_gemm
@pytest.mark.parametrize("m", [2, 4])
@pytest.mark.parametrize(
"n,k",
[
(1024, 4096),
(4096, 4096),
(11008, 4096),
(4096, 11008),
(14336, 4096),
(4096, 14336),
(5120, 5120),
(13824, 5120),
(5120, 13824),
(16384, 4096),
(4096, 16384),
(512, 3584),
(3584, 3584),
(18944, 3584),
(3584, 18944),
(1024, 8192),
(8192, 8192),
(28672, 8192),
(8192, 28672),
(2048, 2048),
(8192, 2048),
(2048, 8192),
],
)
def test_bf16_gemm_matches_common_transformer_shapes(m, n, k):
torch.manual_seed(2026 + m + n + k)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.empty(n, k, device="cuda", dtype=torch.bfloat16)
weight.normal_(mean=0.0, std=0.02)
actual = bf16_gemm(x, weight)
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemm
@pytest.mark.parametrize(
"m,n,k",
[
(1, 8192, 2048),
(8, 4096, 11008),
(8, 512, 3584),
(8, 1024, 8192),
(8, 2048, 8192),
],
)
def test_bf16_gemm_matches_m8_edge_bands(m, n, k):
torch.manual_seed(2026 + m + n + k)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.empty(n, k, device="cuda", dtype=torch.bfloat16)
weight.normal_(mean=0.0, std=0.02)
actual = bf16_gemm(x, weight)
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemm
def test_bf16_gemm_preserves_singleton_batch_and_fuses_bias():
torch.manual_seed(23)
x = torch.randn(1, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
bias = torch.randn(1536, device="cuda", dtype=torch.bfloat16)
actual = bf16_gemm(x, weight, bias)
expected = F.linear(x, weight, bias)
assert actual.shape == (1, 1536)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemm
def test_bf16_gemm_small_batch_fuses_bias():
torch.manual_seed(25)
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
bias = torch.randn(256, device="cuda", dtype=torch.bfloat16)
actual = bf16_gemm(x, weight, bias)
expected = F.linear(x, weight, bias)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemm
def test_bf16_gemm_uses_current_stream():
x = torch.randn(1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
torch.cuda.synchronize()
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
actual = bf16_gemm(x, weight)
expected = F.linear(x, weight)
stream.synchronize()
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemm
def test_bf16_gemm_cuda_graph_replay():
torch.manual_seed(29)
x = torch.randn(1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
for _ in range(3):
bf16_gemm(x, weight)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = bf16_gemm(x, weight)
x.copy_(torch.randn_like(x))
graph.replay()
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemm
@pytest.mark.parametrize("n,k", [(64, 7), (64, 12), (33, 100), (256, 1534)])
def test_bf16_gemm_handles_unaligned_k(n, k):
torch.manual_seed(29)
x = torch.randn(k, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16)
actual = bf16_gemm(x, weight)
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
x3 = torch.randn(3, k, device="cuda", dtype=torch.bfloat16)
actual3 = bf16_gemm(x3, weight)
torch.testing.assert_close(actual3, F.linear(x3, weight), rtol=0.02, atol=0.5)
@skip_no_gemm
@pytest.mark.parametrize("m", [1, 2, 3, 4])
def test_bf16_gemm_accepts_complementary_misalignment(m):
"""Misaligned weight rows plus an x base chosen so the vectorized branch
is entered with a non-16B-aligned ``x`` pointer (regression: the branch
guard checked ``x + whead`` alignment but the uint4 view was rooted at
``x`` itself, faulting with a misaligned-address CUDA error)."""
torch.manual_seed(37)
n, k = 256, 1536
# offset 5 elements = +10 bytes: weight rows land at 10 % 16 (whead=3)
# and x at 10 % 16, so (x + 2*whead) % 16 == 0 selects the fast path.
big_w = torch.randn(n * k + 8, device="cuda", dtype=torch.bfloat16)
weight = big_w[5 : 5 + n * k].view(n, k)
big_x = torch.randn(m * k + 8, device="cuda", dtype=torch.bfloat16)
x = big_x[5 : 5 + m * k].view(m, k) if m > 1 else big_x[5 : 5 + k]
assert (x.data_ptr() & 15) == 10 and (weight.data_ptr() & 15) == 10
actual = bf16_gemm(x, weight)
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.5 if m > 1 else 0.25)
@skip_no_gemm
def test_bf16_gemm_scalar_path_handles_misaligned_weight_only():
"""Weight rows misaligned while x stays 16B-aligned take the scalar-x
middle and must stay exact."""
torch.manual_seed(41)
n, k = 256, 1536
big_w = torch.randn(n * k + 8, device="cuda", dtype=torch.bfloat16)
weight = big_w[5 : 5 + n * k].view(n, k)
x = torch.randn(2, k, device="cuda", dtype=torch.bfloat16)
assert (weight.data_ptr() & 15) == 10 and (x.data_ptr() & 15) == 0
actual = bf16_gemm(x, weight)
torch.testing.assert_close(actual, F.linear(x, weight), rtol=0.02, atol=0.5)
@skip_no_gemm
def test_bf16_gemm_small_batch_cuda_graph_replay():
torch.manual_seed(31)
x = torch.randn(8, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
for _ in range(3):
bf16_gemm(x, weight)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = bf16_gemm(x, weight)
x.copy_(torch.randn_like(x))
graph.replay()
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
@skip_no_gemm
@pytest.mark.parametrize(
"make_args,error",
[
(
lambda: (
torch.randn(65, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
),
"M must",
),
(
lambda: (
torch.randn(16, device="cuda", dtype=torch.float16),
torch.randn(8, 16, device="cuda", dtype=torch.float16),
),
"bf16",
),
(
lambda: (
torch.randn(
16, device="cuda", dtype=torch.bfloat16, requires_grad=True
),
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
),
"autograd",
),
],
)
def test_bf16_gemm_rejects_unsupported_inputs(make_args, error):
with pytest.raises(RuntimeError, match=error):
bf16_gemm(*make_args())
# ---------------------------------------------------------------------------
# Tiled path: M in (8, 64]
# ---------------------------------------------------------------------------
@skip_no_gemm
@pytest.mark.parametrize("m", [9, 12, 16, 17, 24, 32, 33, 48, 64])
@pytest.mark.parametrize("n,k", [(256, 1536), (1536, 1536), (6912, 1536), (1536, 6912)])
def test_bf16_gemm_tiled_matches_decode_batches(m, n, k):
torch.manual_seed(31 + m)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.empty(n, k, device="cuda", dtype=torch.bfloat16)
weight.normal_(mean=0.0, std=0.02)
actual = bf16_gemm(x, weight)
assert actual.shape == (m, n)
_assert_close_fp64(actual, x, weight)
@skip_no_gemm
@pytest.mark.parametrize("m", [12, 64])
def test_bf16_gemm_tiled_matches_lm_head(m):
# N=100000 fills the SMs with N tiles alone: the splits=1 epilogue.
torch.manual_seed(37 + m)
x = torch.randn(m, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.empty(100000, 1536, device="cuda", dtype=torch.bfloat16)
weight.normal_(mean=0.0, std=0.02)
actual = bf16_gemm(x, weight)
_assert_close_fp64(actual, x, weight)
@skip_no_gemm
@pytest.mark.parametrize(
"m,n,k",
[
(12, 100, 72),
(12, 96, 1536),
(12, 1632, 1536),
(64, 100, 72),
(17, 200, 8),
(33, 160, 152),
],
)
def test_bf16_gemm_tiled_handles_remainder_tiles(m, n, k):
# N not a multiple of 64 (predicated epilogue columns) and K not a
# multiple of 64 (zero-filled staging chunks).
torch.manual_seed(41 + m + n + k)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16)
actual = bf16_gemm(x, weight)
_assert_close_fp64(actual, x, weight)
@skip_no_gemm
@pytest.mark.parametrize("m,n,k", [(12, 1536, 1536), (64, 6912, 1536)])
def test_bf16_gemm_tiled_fuses_bias(m, n, k):
# (12, 1536) exercises the narrow-N deep-K config; (64, 6912) the
# wide-N default.
torch.manual_seed(43 + m)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16)
bias = torch.randn(n, device="cuda", dtype=torch.bfloat16)
actual = bf16_gemm(x, weight, bias)
_assert_close_fp64(actual, x, weight, bias)
@skip_no_gemm
def test_bf16_gemm_tiled_deterministic_across_runs():
# Single-pass K accumulation with no atomics: reruns are bitwise
# identical — CUDA Graph replay relies on this.
torch.manual_seed(47)
x = torch.randn(16, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
first = bf16_gemm(x, weight)
second = bf16_gemm(x, weight)
assert torch.equal(first, second)
@skip_no_gemm
def test_bf16_gemm_tiled_cuda_graph_replay():
torch.manual_seed(53)
x = torch.randn(16, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
for _ in range(3):
bf16_gemm(x, weight)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = bf16_gemm(x, weight)
x.copy_(torch.randn_like(x))
graph.replay()
_assert_close_fp64(actual, x, weight)
@skip_no_gemm
def test_bf16_gemm_tiled_rejects_k_not_multiple_of_8():
x = torch.randn(12, 12, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(64, 12, device="cuda", dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="multiple of 8"):
bf16_gemm(x, weight)
@skip_no_gemm
def test_bf16_gemm_tiled_rejects_misaligned_x():
# A 2-byte storage offset breaks the 16B alignment the tiled path
# stages chunks on; the M <= 8 GEMV path still accepts it.
k = 1536
storage = torch.randn(12 * k + 1, device="cuda", dtype=torch.bfloat16)
x8 = storage[1 : 1 + 8 * k].view(8, k)
x12 = storage[1 : 1 + 12 * k].view(12, k)
weight = torch.randn(1536, k, device="cuda", dtype=torch.bfloat16)
torch.testing.assert_close(
bf16_gemm(x8, weight), F.linear(x8, weight), rtol=0.02, atol=0.5
)
with pytest.raises(RuntimeError, match="16-byte"):
bf16_gemm(x12, weight)
tests/extension/test_linear_dispatch.py
-233
View File
@@ -1,233 +0,0 @@
import importlib
import logging
import pytest
import torch
import torch.nn.functional as F
from astrai.extension import is_available, linear
from astrai.extension.dispatch import explain, op_backend, resolve
# The package attribute ``linear`` is the dispatched function; reach the
# module object explicitly for monkeypatching its private helpers.
linear_module = importlib.import_module("astrai.extension.backend.linear")
GEMM_AVAILABLE = (
torch.cuda.is_available()
and is_available("bf16_gemm")
and torch.cuda.get_device_capability() >= (8, 0)
)
skip_no_gemm = pytest.mark.skipif(
not GEMM_AVAILABLE,
reason="BF16 GEMM requires a built kernel and compute capability 8.0+",
)
def _routes_to_gemm(monkeypatch, x, weight, bias=None) -> bool:
"""Patch the GEMM entry point to a sentinel and report whether
``linear`` selected it (torch fallback would compute a real tensor)."""
sentinel = object()
def fake_gemm(x, weight, bias):
return sentinel
monkeypatch.setattr(linear_module, "_inference_bf16_gemm", fake_gemm)
return linear(x, weight, bias) is sentinel
def test_model_linear_routes_through_backend(monkeypatch):
sentinel = torch.randn(2, 4)
def fake_linear(x, weight, bias):
assert x.shape == (2, 3)
assert weight.shape == (4, 3)
assert bias is None
return sentinel
monkeypatch.setattr("astrai.model.components.linear.linear", fake_linear)
from astrai.model.components.linear import Linear
layer = Linear(3, 4)
assert layer(torch.randn(2, 3)) is sentinel
def test_invalid_mode_warns_and_uses_auto(monkeypatch, caplog):
monkeypatch.setenv("ASTRAI_GEMM", "invalid-test-mode")
x = torch.randn(2, 8)
weight = torch.randn(4, 8)
with caplog.at_level(logging.WARNING):
actual = linear(x, weight)
assert "using auto" in caplog.text
torch.testing.assert_close(actual, F.linear(x, weight))
def test_cpu_and_training_calls_fall_back_to_torch(monkeypatch):
monkeypatch.setenv("ASTRAI_GEMM", "1")
x = torch.randn(2, 8, requires_grad=True)
weight = torch.randn(4, 8, requires_grad=True)
actual = linear(x, weight)
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected)
actual.sum().backward()
assert x.grad is not None
assert weight.grad is not None
@skip_no_gemm
@pytest.mark.parametrize("m", [1, 2, 3, 4, 5, 6, 7, 8])
def test_auto_selects_small_decode_batches(monkeypatch, m):
monkeypatch.setenv("ASTRAI_GEMM", "auto")
x = torch.randn(m, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
assert _routes_to_gemm(monkeypatch, x, weight)
@skip_no_gemm
@pytest.mark.parametrize("m", [12, 16, 24, 32])
def test_auto_selects_larger_decode_batches(monkeypatch, m):
"""Auto covers M up to 32; 48+ loses to cuBLAS on long-K shapes."""
monkeypatch.setenv("ASTRAI_GEMM", "auto")
x = torch.randn(m, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
assert _routes_to_gemm(monkeypatch, x, weight)
@skip_no_gemm
@pytest.mark.parametrize("m", [48, 64, 65])
def test_auto_falls_back_outside_band(monkeypatch, m):
"""M beyond 32 falls back to cuBLAS (measured regression at M=48+)."""
monkeypatch.setenv("ASTRAI_GEMM", "auto")
x = torch.randn(m, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
assert not _routes_to_gemm(monkeypatch, x, weight)
torch.testing.assert_close(
linear(x, weight), F.linear(x, weight), rtol=0.02, atol=0.25
)
@skip_no_gemm
def test_mode_zero_disables_gemm(monkeypatch):
monkeypatch.setenv("ASTRAI_GEMM", "0")
x = torch.randn(2, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
assert not _routes_to_gemm(monkeypatch, x, weight)
torch.testing.assert_close(linear(x, weight), F.linear(x, weight))
@skip_no_gemm
@pytest.mark.parametrize("m", [1, 2, 8, 16, 32])
def test_mode_one_forces_every_capable_batch(monkeypatch, m):
monkeypatch.setenv("ASTRAI_GEMM", "1")
x = torch.randn(m, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
assert _routes_to_gemm(monkeypatch, x, weight)
@skip_no_gemm
def test_mode_one_rejects_oversized_batch_and_grad(monkeypatch):
monkeypatch.setenv("ASTRAI_GEMM", "1")
weight = torch.randn(
256, 1536, device="cuda", dtype=torch.bfloat16, requires_grad=True
)
with torch.no_grad():
oversized = torch.randn(65, 1536, device="cuda", dtype=torch.bfloat16)
assert not _routes_to_gemm(monkeypatch, oversized, weight)
assert not _routes_to_gemm(
monkeypatch, torch.randn(2, 1536, device="cuda", dtype=torch.bfloat16), weight
)
@skip_no_gemm
def test_mode_one_supports_bias_and_vector_input(monkeypatch):
monkeypatch.setenv("ASTRAI_GEMM", "1")
x = torch.randn(1, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
bias = torch.randn(256, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
assert _routes_to_gemm(monkeypatch, x, weight, bias)
monkeypatch.undo()
torch.testing.assert_close(
linear(x, weight, bias),
F.linear(x, weight, bias),
rtol=0.02,
atol=0.25,
)
@skip_no_gemm
def test_dispatched_linear_cuda_graph_replay(monkeypatch):
monkeypatch.setenv("ASTRAI_GEMM", "1")
x = torch.randn(1, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
for _ in range(3):
linear(x, weight)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = linear(x, weight)
x.copy_(torch.randn_like(x))
graph.replay()
expected = F.linear(x, weight)
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
def test_linear_family_is_registered_with_shared_dispatcher():
from astrai.extension.dispatch import _FAMILIES
assert "linear" in _FAMILIES
x = torch.randn(2, 8)
weight = torch.randn(4, 8)
resolution = resolve("linear", x, weight)
assert resolution.record.family == "linear"
assert resolution.origin in ("chain", "fallback")
assert "linear" in explain("linear", x, weight)
@skip_no_gemm
def test_ops_env_override_forces_torch_for_capable_call(monkeypatch):
"""ASTR_OPS=linear=torch must keep working after the M-band rewrite
(regression: the family was silently dropped from the dispatcher, so
the override warned, fell through, and the gemm kernel still ran)."""
monkeypatch.setenv("ASTR_OPS", "linear=torch")
x = torch.randn(2, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
assert not _routes_to_gemm(monkeypatch, x, weight)
torch.testing.assert_close(
linear(x, weight), F.linear(x, weight), rtol=0.02, atol=0.25
)
@skip_no_gemm
def test_ops_env_override_forces_gemm(monkeypatch):
monkeypatch.setenv("ASTR_OPS", "linear=gemm")
x = torch.randn(1, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
# M=1 is outside the auto band but inside the forced gemm record.
assert _routes_to_gemm(monkeypatch, x, weight)
@skip_no_gemm
def test_op_backend_context_selects_torch(monkeypatch):
monkeypatch.setenv("ASTRAI_GEMM", "1")
x = torch.randn(2, 1536, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
with torch.no_grad(), op_backend(linear="torch"):
assert not _routes_to_gemm(monkeypatch, x, weight)
torch.testing.assert_close(
linear(x, weight), F.linear(x, weight), rtol=0.02, atol=0.25
)
# The override is scoped: the forced mode applies again afterwards.
with torch.no_grad():
assert _routes_to_gemm(monkeypatch, x, weight)
def test_op_backend_rejects_unknown_linear_handle():
with pytest.raises(ValueError, match="Unknown linear implementation"):
op_backend(linear="nonexistent").__enter__()
tests/extension/test_swiglu.py
-112
View File
@@ -1,112 +0,0 @@
import pytest
import torch
import torch.nn.functional as F
from astrai.extension import bf16_swiglu, is_available
SWIGLU_AVAILABLE = (
torch.cuda.is_available()
and is_available("bf16_swiglu")
and torch.cuda.get_device_capability() >= (8, 0)
)
skip_no_swiglu = pytest.mark.skipif(
not SWIGLU_AVAILABLE,
reason="BF16 SwiGLU requires a built kernel and compute capability 8.0+",
)
def reference_swiglu(x, up_weight, gate_weight):
return F.linear(x, up_weight) * F.silu(F.linear(x, gate_weight))
@skip_no_swiglu
@pytest.mark.parametrize("m", [1, 2, 4, 8])
@pytest.mark.parametrize("n,k", [(6912, 1536), (4096, 4096), (11008, 4096)])
def test_bf16_swiglu_matches_common_dense_mlp_shapes(m, n, k):
torch.manual_seed(37 + m)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
gate_weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
actual = bf16_swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
assert actual.shape == (m, n)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
@skip_no_swiglu
def test_bf16_swiglu_preserves_vector_shape():
x = torch.randn(1536, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight) * 0.02
actual = bf16_swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
assert actual.shape == (256,)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
@skip_no_swiglu
def test_bf16_swiglu_uses_current_stream_and_cuda_graph():
torch.manual_seed(43)
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight) * 0.02
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
for _ in range(3):
bf16_swiglu(x, up_weight, gate_weight)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = bf16_swiglu(x, up_weight, gate_weight)
x.copy_(torch.randn_like(x) * 0.1)
graph.replay()
stream.synchronize()
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
@skip_no_swiglu
@pytest.mark.parametrize(
"make_args,error",
[
(
lambda: (
torch.randn(9, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
),
"M must",
),
(
lambda: (
torch.randn(2, 15, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 15, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 15, device="cuda", dtype=torch.bfloat16),
),
"divisible by 8",
),
(
lambda: (
torch.randn(2, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(8, 16, device="cuda", dtype=torch.bfloat16),
torch.randn(7, 16, device="cuda", dtype=torch.bfloat16),
),
"identical shapes",
),
],
)
def test_bf16_swiglu_rejects_unsupported_inputs(make_args, error):
with pytest.raises(RuntimeError, match=error):
bf16_swiglu(*make_args())
@skip_no_swiglu
def test_bf16_swiglu_rejects_misaligned_storage_with_clear_error():
"""Contiguous-but-offset views must fail the wrapper's TORCH_CHECK with
an actionable message instead of a sticky CUDA misaligned-address error
(regression: the kernel casts x directly to uint4 without checking)."""
k = 1536
base = torch.randn(k + 1, device="cuda", dtype=torch.bfloat16)
x = base[1:] # +2 bytes: contiguous but not 16B-aligned
weights = torch.randn(8, k, device="cuda", dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="16-byte aligned"):
bf16_swiglu(x, weights, weights)
tests/extension/test_swiglu_dispatch.py
-118
View File
@@ -1,118 +0,0 @@
import logging
import pytest
import torch
import torch.nn.functional as F
from astrai.extension import is_available, swiglu
from astrai.model.components.mlp import MLP
SWIGLU_AVAILABLE = (
torch.cuda.is_available()
and is_available("bf16_swiglu")
and torch.cuda.get_device_capability() >= (8, 0)
)
skip_no_swiglu = pytest.mark.skipif(
not SWIGLU_AVAILABLE,
reason="BF16 SwiGLU requires a built kernel and compute capability 8.0+",
)
def reference_swiglu(x, up_weight, gate_weight):
return F.linear(x, up_weight) * F.silu(F.linear(x, gate_weight))
def test_cpu_and_training_calls_fall_back_with_gradients(monkeypatch):
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
x = torch.randn(2, 8, requires_grad=True)
up_weight = torch.randn(4, 8, requires_grad=True)
gate_weight = torch.randn(4, 8, requires_grad=True)
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected)
actual.sum().backward()
assert x.grad is not None
assert up_weight.grad is not None
assert gate_weight.grad is not None
def test_invalid_mode_warns_and_uses_auto(monkeypatch, caplog):
monkeypatch.setenv("ASTRAI_SWIGLU", "invalid-test-mode")
x = torch.randn(2, 8)
up_weight = torch.randn(4, 8)
gate_weight = torch.randn(4, 8)
with caplog.at_level(logging.WARNING):
actual = swiglu(x, up_weight, gate_weight)
assert "using auto" in caplog.text
torch.testing.assert_close(actual, reference_swiglu(x, up_weight, gate_weight))
def test_mlp_routes_through_swiglu_backend(monkeypatch):
sentinel = torch.randn(2, 4)
def fake_swiglu(x, up_weight, gate_weight):
assert x.shape == (2, 3)
assert up_weight.shape == gate_weight.shape == (4, 3)
return sentinel
monkeypatch.setattr("astrai.model.components.mlp.swiglu", fake_swiglu)
layer = MLP(3, 4)
output = layer(torch.randn(2, 3))
assert output["hidden_states"].shape == (2, 3)
@skip_no_swiglu
def test_mode_zero_disables_fused_kernel(monkeypatch):
monkeypatch.setenv("ASTRAI_SWIGLU", "0")
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight) * 0.02
with torch.no_grad():
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected)
@skip_no_swiglu
def test_mode_one_forces_supported_shape(monkeypatch):
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) * 0.1
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight) * 0.02
with torch.no_grad():
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
@skip_no_swiglu
def test_auto_uses_fused_chain_for_decode_batches(monkeypatch):
# Auto adopts the fused primitive for the decode band; numerics match
# the unfused linear-backend chain within BF16 accumulation-order noise.
monkeypatch.setenv("ASTRAI_SWIGLU", "auto")
x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16)
up_weight = torch.randn(6912, 1536, device="cuda", dtype=torch.bfloat16)
gate_weight = torch.randn_like(up_weight)
with torch.no_grad():
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.1)
@skip_no_swiglu
def test_mode_one_falls_back_for_misaligned_storage(monkeypatch):
"""Contiguous-but-offset views must route to the unfused torch chain
even with ASTRAI_SWIGLU=1 instead of reaching the uint4-only kernel
(regression: the fused primitive faulted with a misaligned-address
CUDA error for such inputs)."""
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
k = 1536
x_base = torch.randn(2 * k + 8, device="cuda", dtype=torch.bfloat16) * 0.1
x = x_base[1 : 1 + 2 * k].view(2, k)
assert x.is_contiguous() and (x.data_ptr() & 15) != 0
up_weight = torch.randn(64, k, device="cuda", dtype=torch.bfloat16) * 0.02
gate_weight = torch.randn_like(up_weight)
with torch.no_grad():
actual = swiglu(x, up_weight, gate_weight)
expected = reference_swiglu(x, up_weight, gate_weight)
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)