0z5a
d4a292b36b
- deepen common-shape BF16 GEMV tuning with warp-row tiling for LLaMA/Qwen2/GPT-NeoX/OPT decode projections - add fused BF16 up/gate SwiGLU CUDA primitive with ASTRAI_SWIGLU=0/1/auto dispatch - keep the unfused linear backend as the default path; auto enables no shape until per-architecture checkpoint gates pass - fall back to the linear/torch chain when kernels are absent, on CPU, in training, or outside supported M/K/dtype shapes - add gemv/swiglu benchmark scripts, dispatch and parity tests, and kernel documentation Benchmark: NVIDIA L20 (sm_89), CUDA 12.8, PyTorch 2.11.0+cu128, idle GPU. AstrAI 1B config (24 layers, hidden 1536, vocab 100000), BF16, prompt 128, 32 greedy decode tokens, CUDA graphs enabled, A/B in separate interleaved processes (3 rounds, 8 trials each, medians). Default vs ASTRAI_SWIGLU=1 per generate call: batch 1 134.8->129.1 ms (+4.44%), batch 2 136.2->130.9 ms (+4.06%), batch 4 145.5->140.3 ms (+3.66%). Greedy output identical at batch 1, differs at batch 2/4, so auto stays unfused by default; kernelless fallback verified bit-identical greedy.
32 lines
724 B
Python
32 lines
724 B
Python
"""Backend selection, fallbacks, and execution policies."""
|
|
|
|
from astrai.extension.backend.attention import (
|
|
ATTN_BACKEND,
|
|
AttentionBackend,
|
|
AttentionBackendFactory,
|
|
CudaBackend,
|
|
FlashAttnBackend,
|
|
TorchNativeBackend,
|
|
attention,
|
|
attn_backend,
|
|
get_backend,
|
|
)
|
|
from astrai.extension.backend.linear import linear
|
|
from astrai.extension.backend.rotary import apply_rotary_emb
|
|
from astrai.extension.backend.swiglu import swiglu
|
|
|
|
__all__ = [
|
|
"ATTN_BACKEND",
|
|
"AttentionBackend",
|
|
"AttentionBackendFactory",
|
|
"CudaBackend",
|
|
"FlashAttnBackend",
|
|
"TorchNativeBackend",
|
|
"apply_rotary_emb",
|
|
"attention",
|
|
"attn_backend",
|
|
"get_backend",
|
|
"linear",
|
|
"swiglu",
|
|
]
|