refactor: rework attention backend resolution

- explicit attn_backend() context wins over ASTR_BACKEND env
- polymorphic available()/supports_call() replace isinstance dispatch
- cache singleton backend instances to avoid hot-path allocation
- training (fwd=None) resolves cuda > flash > torch by capability
- flash dense supports mask-free calls only; masked training falls back to torch
This commit is contained in:
2026-08-23 14:47:02 +08:00
parent 10fec8dca1
commit a29bdfae46
6 changed files with 372 additions and 110 deletions
+12 -6
View File
@@ -34,10 +34,13 @@ def _ws(pool: PagePool) -> InferenceWorkspace:
@skip_no_kernel
def test_training_forward_matches_torch(cuda_model):
"""Training forward (kv_cache=None) uses torch-native SDPA.
"""Training forward (kv_cache=None) resolves to a capable dense backend.
CudaBackend does not support training (requires kv_cache).
Torch-native backend must match default (which falls back to torch).
CudaBackend cannot run training (requires a KV cache), so the default
falls back by capability — flash when it can handle the call
(mask-free/causal), otherwise torch SDPA. The default path must not
raise and must produce finite logits; explicitly selected torch SDPA
must be deterministic across runs.
"""
model, _ = cuda_model
@@ -48,12 +51,15 @@ def test_training_forward_matches_torch(cuda_model):
with attn_backend(ATTN_BACKEND.TORCH_NATIVE):
with torch.no_grad():
out_torch = model(input_ids)
out_torch_a = model(input_ids)
with torch.no_grad():
out_torch_b = model(input_ids)
assert out_default["logits"].shape == out_torch_a["logits"].shape
assert torch.isfinite(out_default["logits"]).all()
torch.testing.assert_close(
out_torch["logits"], out_default["logits"], atol=1e-6, rtol=1e-6
out_torch_a["logits"], out_torch_b["logits"], atol=0, rtol=0
)
assert out_default["logits"].shape[0] == 2
@skip_no_kernel