0z5a
d4a292b36b
- deepen common-shape BF16 GEMV tuning with warp-row tiling for LLaMA/Qwen2/GPT-NeoX/OPT decode projections - add fused BF16 up/gate SwiGLU CUDA primitive with ASTRAI_SWIGLU=0/1/auto dispatch - keep the unfused linear backend as the default path; auto enables no shape until per-architecture checkpoint gates pass - fall back to the linear/torch chain when kernels are absent, on CPU, in training, or outside supported M/K/dtype shapes - add gemv/swiglu benchmark scripts, dispatch and parity tests, and kernel documentation Benchmark: NVIDIA L20 (sm_89), CUDA 12.8, PyTorch 2.11.0+cu128, idle GPU. AstrAI 1B config (24 layers, hidden 1536, vocab 100000), BF16, prompt 128, 32 greedy decode tokens, CUDA graphs enabled, A/B in separate interleaved processes (3 rounds, 8 trials each, medians). Default vs ASTRAI_SWIGLU=1 per generate call: batch 1 134.8->129.1 ms (+4.44%), batch 2 136.2->130.9 ms (+4.06%), batch 4 145.5->140.3 ms (+3.66%). Greedy output identical at batch 1, differs at batch 2/4, so auto stays unfused by default; kernelless fallback verified bit-identical greedy.
94 lines
2.1 KiB
Python
94 lines
2.1 KiB
Python
"""CUDA attention kernel wrappers with torch fallback.
|
|
|
|
Public API:
|
|
- ``attn_decode`` — single-query decode attention
|
|
- ``attn_prefill`` — multi-query prefill attention
|
|
- ``attn_paged_decode`` — paged decode attention (direct page-table access)
|
|
- ``AttentionBackend`` — ABC for attention computation strategies
|
|
- ``TorchNativeBackend`` — default SDPA backend with KV cache I/O
|
|
- ``CudaBackend`` — CUDA kernel backend with paged decode + prefill
|
|
|
|
Layout convention: all q/k/v are ``[batch, seq_len, n_heads, head_dim]``
|
|
(blhd). Scale is always ``1/sqrt(head_dim)``.
|
|
|
|
Each wrapper calls its compiled CUDA kernel directly. Fallback to torch
|
|
SDPA is handled by the attention backend, not the wrapper functions.
|
|
"""
|
|
|
|
from astrai.extension.backend import (
|
|
ATTN_BACKEND,
|
|
AttentionBackend,
|
|
AttentionBackendFactory,
|
|
CudaBackend,
|
|
FlashAttnBackend,
|
|
TorchNativeBackend,
|
|
apply_rotary_emb,
|
|
attention,
|
|
attn_backend,
|
|
get_backend,
|
|
linear,
|
|
swiglu,
|
|
)
|
|
from astrai.extension.dispatch import (
|
|
Axes,
|
|
ExplicitSelectionError,
|
|
ImplRecord,
|
|
Resolution,
|
|
Spec,
|
|
axis,
|
|
explain,
|
|
explain_plan,
|
|
op_backend,
|
|
register_env_alias,
|
|
register_family,
|
|
resolve,
|
|
resolve_plan,
|
|
tensor_axes,
|
|
)
|
|
from astrai.extension.loader import KERNEL_NAMES, is_available
|
|
from astrai.extension.ops import (
|
|
TensorLayout,
|
|
attn_decode,
|
|
attn_paged_decode,
|
|
attn_prefill,
|
|
bf16_gemv,
|
|
bf16_swiglu,
|
|
)
|
|
|
|
__all__ = [
|
|
"ATTN_BACKEND",
|
|
"AttentionBackend",
|
|
"AttentionBackendFactory",
|
|
"CudaBackend",
|
|
"TorchNativeBackend",
|
|
"FlashAttnBackend",
|
|
"TensorLayout",
|
|
"attention",
|
|
"attn_backend",
|
|
"get_backend",
|
|
"linear",
|
|
"swiglu",
|
|
"attn_decode",
|
|
"attn_paged_decode",
|
|
"attn_prefill",
|
|
"bf16_gemv",
|
|
"bf16_swiglu",
|
|
"is_available",
|
|
"KERNEL_NAMES",
|
|
"apply_rotary_emb",
|
|
"Axes",
|
|
"ExplicitSelectionError",
|
|
"ImplRecord",
|
|
"Resolution",
|
|
"Spec",
|
|
"axis",
|
|
"explain",
|
|
"explain_plan",
|
|
"op_backend",
|
|
"register_env_alias",
|
|
"register_family",
|
|
"resolve",
|
|
"resolve_plan",
|
|
"tensor_axes",
|
|
]
|