feat: auto-select best available attention backend

- Default backend resolves to highest-priority available: flash -> cuda -> torch
- attention() falls back per-call for training/fp32/unsupported head_dim
- Re-apply index_copy_ for CUDA KV cache writes (index_put_ race mitigation)
This commit is contained in:
2026-08-07 13:48:59 +08:00
parent a59ae8f32e
commit 6f49738991
3 changed files with 105 additions and 22 deletions
+9 -7
View File
@@ -27,9 +27,9 @@ def _ws(pool: PagePool) -> InferenceWorkspace:
def test_training_forward_matches_torch(cuda_model):
"""Training forward (kv_cache=None) should produce identical logits.
CudaBackend is inference-only: it raises when kv_cache is None. Training
must use TorchNativeBackend (the default). Verify the torch path is
stable and that CudaBackend rejects the training path explicitly.
CudaBackend is now safe as a default: for training (``kv_cache=None``)
or non-bf16 inputs it falls back to torch SDPA. Verify the fallback
path matches the torch-native forward exactly.
"""
import pytest
@@ -39,11 +39,13 @@ def test_training_forward_matches_torch(cuda_model):
with torch.no_grad():
out_torch = model(input_ids)
with pytest.raises(RuntimeError, match="does not support training"):
with attn_backend(ATTN_BACKEND.CUDA):
with torch.no_grad():
model(input_ids)
with attn_backend(ATTN_BACKEND.CUDA):
with torch.no_grad():
out_cuda = model(input_ids)
torch.testing.assert_close(
out_cuda["logits"], out_torch["logits"], atol=1e-6, rtol=1e-6
)
assert out_torch["logits"].shape[0] == 2