fix: resolve audited dispatch, kernel, and rollout bugs
- re-register the linear family with the operator dispatcher (ASTR_OPS / op_backend / resolve) - fix bf16 gemv misaligned-address faults and element mispairing for offset weights - reject misaligned bf16_swiglu inputs with a clear error and fall back in the backend gate - make the rollout reuse decision, validation, and return atomic under one policy snapshot - add the documented post-scoring rollout version check - derive live+1 under the scheduler lock in optimizer_step via apply_weight_update(None, ...) - reject rollout_max_policy_lag below rollout_interval - 1 at config time - sync gemv stream-test inputs before switching streams; drop dead loader imports
This commit is contained in:
@@ -130,6 +130,7 @@ def test_bf16_gemv_small_batch_fuses_bias():
|
||||
def test_bf16_gemv_uses_current_stream():
|
||||
x = torch.randn(1536, device="cuda", dtype=torch.bfloat16)
|
||||
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
torch.cuda.synchronize()
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
actual = bf16_gemv(x, weight)
|
||||
@@ -171,6 +172,43 @@ def test_bf16_gemv_handles_unaligned_k(n, k):
|
||||
torch.testing.assert_close(actual3, F.linear(x3, weight), rtol=0.02, atol=0.5)
|
||||
|
||||
|
||||
@skip_no_gemv
|
||||
@pytest.mark.parametrize("m", [1, 2, 3, 4])
|
||||
def test_bf16_gemv_accepts_complementary_misalignment(m):
|
||||
"""Misaligned weight rows plus an x base chosen so the vectorized branch
|
||||
is entered with a non-16B-aligned ``x`` pointer (regression: the branch
|
||||
guard checked ``x + whead`` alignment but the uint4 view was rooted at
|
||||
``x`` itself, faulting with a misaligned-address CUDA error)."""
|
||||
torch.manual_seed(37)
|
||||
n, k = 256, 1536
|
||||
# offset 5 elements = +10 bytes: weight rows land at 10 % 16 (whead=3)
|
||||
# and x at 10 % 16, so (x + 2*whead) % 16 == 0 selects the fast path.
|
||||
big_w = torch.randn(n * k + 8, device="cuda", dtype=torch.bfloat16)
|
||||
weight = big_w[5 : 5 + n * k].view(n, k)
|
||||
big_x = torch.randn(m * k + 8, device="cuda", dtype=torch.bfloat16)
|
||||
x = big_x[5 : 5 + m * k].view(m, k) if m > 1 else big_x[5 : 5 + k]
|
||||
assert (x.data_ptr() & 15) == 10 and (weight.data_ptr() & 15) == 10
|
||||
|
||||
actual = bf16_gemv(x, weight)
|
||||
expected = F.linear(x, weight)
|
||||
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.5 if m > 1 else 0.25)
|
||||
|
||||
|
||||
@skip_no_gemv
|
||||
def test_bf16_gemv_scalar_path_handles_misaligned_weight_only():
|
||||
"""Weight rows misaligned while x stays 16B-aligned take the scalar-x
|
||||
middle and must stay exact."""
|
||||
torch.manual_seed(41)
|
||||
n, k = 256, 1536
|
||||
big_w = torch.randn(n * k + 8, device="cuda", dtype=torch.bfloat16)
|
||||
weight = big_w[5 : 5 + n * k].view(n, k)
|
||||
x = torch.randn(2, k, device="cuda", dtype=torch.bfloat16)
|
||||
assert (weight.data_ptr() & 15) == 10 and (x.data_ptr() & 15) == 0
|
||||
|
||||
actual = bf16_gemv(x, weight)
|
||||
torch.testing.assert_close(actual, F.linear(x, weight), rtol=0.02, atol=0.5)
|
||||
|
||||
|
||||
@skip_no_gemv
|
||||
def test_bf16_gemv_small_batch_cuda_graph_replay():
|
||||
torch.manual_seed(31)
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch.nn.functional as F
|
||||
|
||||
from astrai.extension import is_available, linear
|
||||
from astrai.extension.backend import linear as public_linear
|
||||
from astrai.extension.dispatch import explain, op_backend, resolve
|
||||
|
||||
# The package attribute ``linear`` is the dispatched function; reach the
|
||||
# module object explicitly for monkeypatching its private helpers.
|
||||
@@ -166,3 +167,60 @@ def test_dispatched_linear_cuda_graph_replay(monkeypatch):
|
||||
graph.replay()
|
||||
expected = F.linear(x, weight)
|
||||
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.25)
|
||||
|
||||
|
||||
def test_linear_family_is_registered_with_shared_dispatcher():
|
||||
from astrai.extension.dispatch import _FAMILIES
|
||||
|
||||
assert "linear" in _FAMILIES
|
||||
x = torch.randn(2, 8)
|
||||
weight = torch.randn(4, 8)
|
||||
resolution = resolve("linear", x, weight)
|
||||
assert resolution.record.family == "linear"
|
||||
assert resolution.origin in ("chain", "fallback")
|
||||
assert "linear" in explain("linear", x, weight)
|
||||
|
||||
|
||||
@skip_no_gemv
|
||||
def test_ops_env_override_forces_torch_for_capable_call(monkeypatch):
|
||||
"""ASTR_OPS=linear=torch must keep working after the M-band rewrite
|
||||
(regression: the family was silently dropped from the dispatcher, so
|
||||
the override warned, fell through, and the gemv kernel still ran)."""
|
||||
monkeypatch.setenv("ASTR_OPS", "linear=torch")
|
||||
x = torch.randn(2, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
weight = torch.randn(1536, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
with torch.no_grad():
|
||||
assert not _routes_to_gemv(monkeypatch, x, weight)
|
||||
torch.testing.assert_close(
|
||||
linear(x, weight), F.linear(x, weight), rtol=0.02, atol=0.25
|
||||
)
|
||||
|
||||
|
||||
@skip_no_gemv
|
||||
def test_ops_env_override_forces_gemv(monkeypatch):
|
||||
monkeypatch.setenv("ASTR_OPS", "linear=gemv")
|
||||
x = torch.randn(1, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
with torch.no_grad():
|
||||
# M=1 is outside the auto band but inside the forced gemv record.
|
||||
assert _routes_to_gemv(monkeypatch, x, weight)
|
||||
|
||||
|
||||
@skip_no_gemv
|
||||
def test_op_backend_context_selects_torch(monkeypatch):
|
||||
monkeypatch.setenv("ASTRAI_GEMV", "1")
|
||||
x = torch.randn(2, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
weight = torch.randn(256, 1536, device="cuda", dtype=torch.bfloat16)
|
||||
with torch.no_grad(), op_backend(linear="torch"):
|
||||
assert not _routes_to_gemv(monkeypatch, x, weight)
|
||||
torch.testing.assert_close(
|
||||
linear(x, weight), F.linear(x, weight), rtol=0.02, atol=0.25
|
||||
)
|
||||
# The override is scoped: the forced mode applies again afterwards.
|
||||
with torch.no_grad():
|
||||
assert _routes_to_gemv(monkeypatch, x, weight)
|
||||
|
||||
|
||||
def test_op_backend_rejects_unknown_linear_handle():
|
||||
with pytest.raises(ValueError, match="Unknown linear implementation"):
|
||||
op_backend(linear="nonexistent").__enter__()
|
||||
|
||||
@@ -97,3 +97,16 @@ def test_bf16_swiglu_uses_current_stream_and_cuda_graph():
|
||||
def test_bf16_swiglu_rejects_unsupported_inputs(make_args, error):
|
||||
with pytest.raises(RuntimeError, match=error):
|
||||
bf16_swiglu(*make_args())
|
||||
|
||||
|
||||
@skip_no_swiglu
|
||||
def test_bf16_swiglu_rejects_misaligned_storage_with_clear_error():
|
||||
"""Contiguous-but-offset views must fail the wrapper's TORCH_CHECK with
|
||||
an actionable message instead of a sticky CUDA misaligned-address error
|
||||
(regression: the kernel casts x directly to uint4 without checking)."""
|
||||
k = 1536
|
||||
base = torch.randn(k + 1, device="cuda", dtype=torch.bfloat16)
|
||||
x = base[1:] # +2 bytes: contiguous but not 16B-aligned
|
||||
weights = torch.randn(8, k, device="cuda", dtype=torch.bfloat16)
|
||||
with pytest.raises(RuntimeError, match="16-byte aligned"):
|
||||
bf16_swiglu(x, weights, weights)
|
||||
|
||||
@@ -95,3 +95,22 @@ def test_auto_uses_unfused_chain_until_shape_is_qualified(monkeypatch):
|
||||
actual = swiglu(x, up_weight, gate_weight)
|
||||
expected = reference_swiglu(x, up_weight, gate_weight)
|
||||
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.1)
|
||||
|
||||
|
||||
@skip_no_swiglu
|
||||
def test_mode_one_falls_back_for_misaligned_storage(monkeypatch):
|
||||
"""Contiguous-but-offset views must route to the unfused torch chain
|
||||
even with ASTRAI_SWIGLU=1 instead of reaching the uint4-only kernel
|
||||
(regression: the fused primitive faulted with a misaligned-address
|
||||
CUDA error for such inputs)."""
|
||||
monkeypatch.setenv("ASTRAI_SWIGLU", "1")
|
||||
k = 1536
|
||||
x_base = torch.randn(2 * k + 8, device="cuda", dtype=torch.bfloat16) * 0.1
|
||||
x = x_base[1 : 1 + 2 * k].view(2, k)
|
||||
assert x.is_contiguous() and (x.data_ptr() & 15) != 0
|
||||
up_weight = torch.randn(64, k, device="cuda", dtype=torch.bfloat16) * 0.02
|
||||
gate_weight = torch.randn_like(up_weight)
|
||||
with torch.no_grad():
|
||||
actual = swiglu(x, up_weight, gate_weight)
|
||||
expected = reference_swiglu(x, up_weight, gate_weight)
|
||||
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.01)
|
||||
|
||||
Reference in New Issue
Block a user