ViperEkura
3067a8e1a6
- Add attention() functional entry delegating to active backend - GQA/MLA forward calls attention() instead of inline cache/SDPA - CUDA kernels support 2D/3D/4D mask via mask_h_stride field - CudaBackend.fwd_decode builds 2D padding mask for mixed seq_lens - KVCache.max_len precomputed in bind_tasks to avoid GPU sync - batch==1 decode short-circuits mask=None - Split tests into conftest, test_backend, test_backend_equivalence, test_kernel_mask - 440 tests pass, L20 decode 1.44-1.60x speedup vs torch native
36 lines
886 B
Python
36 lines
886 B
Python
"""Shared fixtures for extension tests."""
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from astrai.config.model_config import AutoRegressiveLMConfig
|
|
from astrai.extension import is_available
|
|
from astrai.model.transformer import AutoRegressiveLM
|
|
|
|
CUDA_AVAILABLE = torch.cuda.is_available() and is_available("attn_paged_decode")
|
|
skip_no_cuda = pytest.mark.skipif(
|
|
not CUDA_AVAILABLE, reason="CUDA not available or kernels not built"
|
|
)
|
|
|
|
D = 64
|
|
CFG = dict(
|
|
vocab_size=1000,
|
|
hidden_size=128,
|
|
num_attention_heads=2,
|
|
num_key_value_heads=1,
|
|
intermediate_size=256,
|
|
max_position_embeddings=64,
|
|
num_hidden_layers=2,
|
|
rms_norm_eps=1e-5,
|
|
attn_type="gqa",
|
|
ffn_type="mlp",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def cuda_model():
|
|
config = AutoRegressiveLMConfig(**CFG)
|
|
model = AutoRegressiveLM(config).to(device="cuda", dtype=torch.bfloat16)
|
|
model.eval()
|
|
return model, config
|