perf: tune bf16 gemv and add opt-in fused swiglu
- deepen common-shape BF16 GEMV tuning with warp-row tiling for LLaMA/Qwen2/GPT-NeoX/OPT decode projections - add fused BF16 up/gate SwiGLU CUDA primitive with ASTRAI_SWIGLU=0/1/auto dispatch - keep the unfused linear backend as the default path; auto enables no shape until per-architecture checkpoint gates pass - fall back to the linear/torch chain when kernels are absent, on CPU, in training, or outside supported M/K/dtype shapes - add gemv/swiglu benchmark scripts, dispatch and parity tests, and kernel documentation Benchmark: NVIDIA L20 (sm_89), CUDA 12.8, PyTorch 2.11.0+cu128, idle GPU. AstrAI 1B config (24 layers, hidden 1536, vocab 100000), BF16, prompt 128, 32 greedy decode tokens, CUDA graphs enabled, A/B in separate interleaved processes (3 rounds, 8 trials each, medians). Default vs ASTRAI_SWIGLU=1 per generate call: batch 1 134.8->129.1 ms (+4.44%), batch 2 136.2->130.9 ms (+4.06%), batch 4 145.5->140.3 ms (+3.66%). Greedy output identical at batch 1, differs at batch 2/4, so auto stays unfused by default; kernelless fallback verified bit-identical greedy.
This commit is contained in:
0z5a
@@ -13,6 +13,7 @@ from astrai.extension.backend.attention import (
|
||||
)
|
||||
from astrai.extension.backend.linear import linear
|
||||
from astrai.extension.backend.rotary import apply_rotary_emb
|
||||
from astrai.extension.backend.swiglu import swiglu
|
||||
|
||||
__all__ = [
|
||||
"ATTN_BACKEND",
|
||||
@@ -26,4 +27,5 @@ __all__ = [
|
||||
"attn_backend",
|
||||
"get_backend",
|
||||
"linear",
|
||||
"swiglu",
|
||||
]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
"""Inference-only dispatch for AstrAI linear layers.
|
||||
|
||||
The CUDA GEMV path is deliberately narrow: automatic selection is enabled
|
||||
only for single-row BF16 shapes measured to beat ``F.linear`` on a supported
|
||||
architecture. Every training, prefill, unsupported-layout, and unmeasured
|
||||
call falls back to PyTorch.
|
||||
only for small decode batches and BF16 shapes measured to beat ``F.linear``
|
||||
on a supported architecture. Every training, prefill, unsupported-layout,
|
||||
and unmeasured call falls back to PyTorch.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -30,20 +30,72 @@ from astrai.extension.ops.gemv import bf16_gemv
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Shape keys are (N, K) for Y[M, N] = X[M, K] @ W[N, K].T. A band is
|
||||
# automatic only after both the per-shape >=5% and end-to-end decode >=3%
|
||||
# gates pass and checkpoint greedy output remains stable. M=1 and M=8 remain
|
||||
# empty on SM89; the safe M=2/4 bands improve real-engine throughput by
|
||||
# 11.8-14.0%.
|
||||
# automatic only after both the per-shape >=5% and projection-chain/engine
|
||||
# >=3% gates pass and output argmax remains stable. M=1 is limited to OPT 1.3B;
|
||||
# M=8 remains empty because at least one projection in each measured family
|
||||
# misses the per-shape gate even when its aggregate chain result is positive.
|
||||
_COMMON_TRANSFORMER_SM89_SHAPES = frozenset(
|
||||
{
|
||||
(1024, 4096), # LLaMA 3 8B K/V
|
||||
(4096, 4096), # LLaMA 2/3 7B/8B Q/O
|
||||
(11008, 4096), # LLaMA 2 7B gate/up
|
||||
(4096, 11008), # LLaMA 2 7B down
|
||||
(14336, 4096), # LLaMA 3 8B gate/up
|
||||
(4096, 14336), # LLaMA 3 8B down
|
||||
(5120, 5120), # LLaMA 2 13B Q/K/V/O
|
||||
(13824, 5120), # LLaMA 2 13B gate/up
|
||||
(5120, 13824), # LLaMA 2 13B down
|
||||
(16384, 4096), # GPT-NeoX MLP up
|
||||
(4096, 16384), # GPT-NeoX MLP down
|
||||
}
|
||||
)
|
||||
_COMMON_TRANSFORMER_SM89_M4_SHAPES = _COMMON_TRANSFORMER_SM89_SHAPES - {
|
||||
(4096, 4096),
|
||||
(11008, 4096),
|
||||
(4096, 11008),
|
||||
}
|
||||
_QWEN2_7B_SM89_SHAPES = frozenset(
|
||||
{
|
||||
(512, 3584), # K/V
|
||||
(3584, 3584), # Q/O
|
||||
(18944, 3584), # gate/up
|
||||
(3584, 18944), # down
|
||||
}
|
||||
)
|
||||
_LLAMA3_70B_SM89_SHAPES = frozenset(
|
||||
{
|
||||
(1024, 8192), # K/V
|
||||
(8192, 8192), # Q/O
|
||||
(28672, 8192), # gate/up
|
||||
(8192, 28672), # down
|
||||
}
|
||||
)
|
||||
_OPT_1_3B_SM89_SHAPES = frozenset(
|
||||
{
|
||||
(2048, 2048), # Q/K/V/O
|
||||
(8192, 2048), # MLP up
|
||||
(2048, 8192), # MLP down
|
||||
}
|
||||
)
|
||||
|
||||
_AUTO_GEMV_SHAPES: dict[tuple[int, int], dict[int, frozenset[tuple[int, int]]]] = {
|
||||
(8, 9): {
|
||||
2: frozenset(
|
||||
1: _OPT_1_3B_SM89_SHAPES,
|
||||
2: _COMMON_TRANSFORMER_SM89_SHAPES
|
||||
| _QWEN2_7B_SM89_SHAPES
|
||||
| _LLAMA3_70B_SM89_SHAPES
|
||||
| _OPT_1_3B_SM89_SHAPES
|
||||
| frozenset(
|
||||
{
|
||||
(256, 1536),
|
||||
(1536, 1536),
|
||||
(100000, 1536),
|
||||
}
|
||||
),
|
||||
4: frozenset({(256, 1536), (1536, 1536)}),
|
||||
4: _COMMON_TRANSFORMER_SM89_M4_SHAPES
|
||||
| _QWEN2_7B_SM89_SHAPES
|
||||
| _LLAMA3_70B_SM89_SHAPES
|
||||
| frozenset({(256, 1536), (1536, 1536)}),
|
||||
}
|
||||
}
|
||||
_AUTO_GEMV_M = frozenset(
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Inference-only fused SwiGLU selection for dense MLP layers."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
|
||||
from astrai.extension.backend.linear import linear
|
||||
from astrai.extension.loader import is_available
|
||||
from astrai.extension.ops.swiglu import bf16_swiglu
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Shape keys are (N, K) for the paired up/gate projections. Automatic entries
|
||||
# are populated only after the primitive, MLP chain, and greedy checkpoint
|
||||
# gates pass on that architecture.
|
||||
_AUTO_SWIGLU_SHAPES: dict[tuple[int, int], dict[int, frozenset[tuple[int, int]]]] = {}
|
||||
_AUTO_SWIGLU_M = frozenset(
|
||||
m for architecture in _AUTO_SWIGLU_SHAPES.values() for m in architecture
|
||||
)
|
||||
_VALID_MODES = {"0", "1", "auto"}
|
||||
_WARNED_MODES: set[str] = set()
|
||||
|
||||
|
||||
def _swiglu_mode() -> str:
|
||||
mode = os.environ.get("ASTRAI_SWIGLU", "auto").strip().lower()
|
||||
if mode in _VALID_MODES:
|
||||
return mode
|
||||
if mode not in _WARNED_MODES:
|
||||
_WARNED_MODES.add(mode)
|
||||
logger.warning(
|
||||
"ASTRAI_SWIGLU=%r is invalid; expected 0, 1, or auto; using auto",
|
||||
mode,
|
||||
)
|
||||
return "auto"
|
||||
|
||||
|
||||
def _unfused_swiglu(x: Tensor, up_weight: Tensor, gate_weight: Tensor) -> Tensor:
|
||||
# Keep the existing linear backend in the fallback chain. This preserves
|
||||
# any independently qualified GEMV shapes instead of making the fusion
|
||||
# decision suppress linear-level optimizations.
|
||||
return linear(x, up_weight) * F.silu(linear(x, gate_weight))
|
||||
|
||||
|
||||
def _fused_swiglu(x: Tensor, up_weight: Tensor, gate_weight: Tensor) -> Tensor:
|
||||
return bf16_swiglu(x.detach(), up_weight.detach(), gate_weight.detach())
|
||||
|
||||
|
||||
@cache
|
||||
def _device_capability(device_index: int) -> tuple[int, int]:
|
||||
return torch.cuda.get_device_capability(device_index)
|
||||
|
||||
|
||||
def _swiglu_capable(x: Tensor, up_weight: Tensor, gate_weight: Tensor) -> bool:
|
||||
return not (
|
||||
torch.is_grad_enabled()
|
||||
or not x.is_cuda
|
||||
or x.dtype != torch.bfloat16
|
||||
or up_weight.dtype != torch.bfloat16
|
||||
or gate_weight.dtype != torch.bfloat16
|
||||
or x.ndim not in (1, 2)
|
||||
or up_weight.ndim != 2
|
||||
or gate_weight.ndim != 2
|
||||
or (x.ndim == 2 and not 1 <= x.shape[0] <= 8)
|
||||
or up_weight.shape != gate_weight.shape
|
||||
or x.shape[-1] != up_weight.shape[1]
|
||||
or x.shape[-1] % 8 != 0
|
||||
or x.device != up_weight.device
|
||||
or x.device != gate_weight.device
|
||||
or not x.is_contiguous()
|
||||
or not up_weight.is_contiguous()
|
||||
or not gate_weight.is_contiguous()
|
||||
or not is_available("bf16_swiglu")
|
||||
)
|
||||
|
||||
|
||||
def _auto_swiglu_shape(x: Tensor, up_weight: Tensor) -> bool:
|
||||
capability = _device_capability(x.get_device())
|
||||
m = 1 if x.ndim == 1 else x.shape[0]
|
||||
return (up_weight.shape[0], up_weight.shape[1]) in _AUTO_SWIGLU_SHAPES.get(
|
||||
capability, {}
|
||||
).get(m, ())
|
||||
|
||||
|
||||
def swiglu(x: Tensor, up_weight: Tensor, gate_weight: Tensor) -> Tensor:
|
||||
"""Apply the dense-MLP SwiGLU projection with a safe torch fallback.
|
||||
|
||||
``ASTRAI_SWIGLU=0`` keeps the unfused linear-backend chain, ``1`` forces
|
||||
the fused primitive for supported inputs, and ``auto`` uses only
|
||||
architecture/shape bands backed by benchmark and checkpoint evidence.
|
||||
"""
|
||||
mode = _swiglu_mode()
|
||||
if mode == "0" or (mode == "auto" and not _AUTO_SWIGLU_SHAPES):
|
||||
return _unfused_swiglu(x, up_weight, gate_weight)
|
||||
if mode == "auto":
|
||||
m = 1 if x.ndim == 1 else (x.shape[0] if x.ndim == 2 else None)
|
||||
if m not in _AUTO_SWIGLU_M:
|
||||
return _unfused_swiglu(x, up_weight, gate_weight)
|
||||
if _swiglu_capable(x, up_weight, gate_weight) and (
|
||||
mode == "1" or _auto_swiglu_shape(x, up_weight)
|
||||
):
|
||||
return _fused_swiglu(x, up_weight, gate_weight)
|
||||
return _unfused_swiglu(x, up_weight, gate_weight)
|
||||
|
||||
|
||||
__all__ = ["swiglu"]
|
||||
Reference in New Issue
Block a user