247 lines
7.4 KiB
Python
247 lines
7.4 KiB
Python
"""GQA attention wrapper functions — one entry point per compiled kernel.
|
|
|
|
Each wrapper dispatches to its CUDA kernel (loaded in ``loader.py``) when
|
|
available, otherwise falls back to ``torch`` SDPA.
|
|
|
|
Interface (all functions):
|
|
causal_offset: -1 = non-causal; >=0 = absolute position of first Q token
|
|
mask: 2D [batch, kv_len] or 3D [batch, q_len, kv_len] (bool)
|
|
scale: 0.0 = auto (1/sqrt(head_dim)); >0 = explicit
|
|
layout: "bhld" (default) or "blhd"
|
|
|
|
Add new kernel wrappers here; split into per-variant files only if this file
|
|
grows large.
|
|
"""
|
|
|
|
import math
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
from astrai.extension.loader import _available, _modules
|
|
|
|
_LAYOUT_CODES: dict[str, int] = {"bhld": 0, "blhd": 1}
|
|
|
|
|
|
def _parse_layout(layout: str | int) -> int:
|
|
if isinstance(layout, int):
|
|
return layout
|
|
code = _LAYOUT_CODES.get(layout.lower())
|
|
if code is None:
|
|
raise ValueError(
|
|
f"unknown layout '{layout}', expected one of {list(_LAYOUT_CODES)}"
|
|
)
|
|
return code
|
|
|
|
|
|
def _to_bhld(t: torch.Tensor, layout: int) -> torch.Tensor:
|
|
"""Normalize to b h l d view. Zero-copy transpose if layout==1 (b l h d)."""
|
|
if layout == 1:
|
|
return t.transpose(1, 2)
|
|
return t
|
|
|
|
|
|
def _expand_kv_heads(
|
|
k: torch.Tensor, v: torch.Tensor, q_head: int
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Expand K/V heads to match Q heads for GQA fallback."""
|
|
kv_head = k.size(1)
|
|
if kv_head == q_head:
|
|
return k, v
|
|
group = q_head // kv_head
|
|
k = k.repeat_interleave(group, dim=1)
|
|
v = v.repeat_interleave(group, dim=1)
|
|
return k, v
|
|
|
|
|
|
def _build_attn_mask(
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
mask: torch.Tensor | None,
|
|
causal_offset: int,
|
|
scale: float,
|
|
) -> tuple[torch.Tensor | None, float]:
|
|
"""Build SDPA-compatible attn_mask + resolved scale.
|
|
|
|
q and k must already be in b h l d layout.
|
|
Causal and mask can coexist: causal sets -inf above the diagonal, mask
|
|
sets -inf for padded positions. Both are OR'd into a single bool mask.
|
|
"""
|
|
q_len = q.size(2)
|
|
kv_len = k.size(2)
|
|
head_dim = q.size(3)
|
|
resolved_scale = scale if scale and scale > 0 else 1.0 / math.sqrt(head_dim)
|
|
|
|
attn_mask = None
|
|
|
|
if mask is not None:
|
|
if mask.dim() == 2:
|
|
# [batch, kv_len] → [batch, 1, 1, kv_len]
|
|
attn_mask = mask[:, None, None, :]
|
|
elif mask.dim() == 3:
|
|
# [batch, q_len, kv_len] → [batch, 1, q_len, kv_len]
|
|
attn_mask = mask[:, None, :, :]
|
|
else:
|
|
raise ValueError(f"mask must be 2D or 3D, got {mask.dim()}D")
|
|
|
|
if causal_offset >= 0:
|
|
batch = q.size(0)
|
|
# q row i attends to kv cols 0..(causal_offset + i)
|
|
q_idx = torch.arange(q_len, device=q.device).unsqueeze(1) # [q_len, 1]
|
|
kv_idx = torch.arange(kv_len, device=q.device).unsqueeze(0) # [1, kv_len]
|
|
causal_bool = kv_idx > (causal_offset + q_idx) # True = masked out
|
|
causal_mask = causal_bool.unsqueeze(0).expand(
|
|
batch, -1, -1
|
|
) # [batch, q_len, kv_len]
|
|
causal_mask = causal_mask[:, None, :, :] # [batch, 1, q_len, kv_len]
|
|
|
|
if attn_mask is not None:
|
|
attn_mask = attn_mask | causal_mask
|
|
else:
|
|
attn_mask = causal_mask
|
|
|
|
return attn_mask, resolved_scale
|
|
|
|
|
|
def _torch_fallback(
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
v: torch.Tensor,
|
|
mask: torch.Tensor | None,
|
|
causal_offset: int,
|
|
scale: float,
|
|
q_layout: int,
|
|
kv_layout: int | None = None,
|
|
) -> torch.Tensor:
|
|
"""Reference attention via ``scaled_dot_product_attention``.
|
|
|
|
q_layout / kv_layout: 0 = b h l d, 1 = b l h d.
|
|
If kv_layout is None, uses q_layout (Q and K/V share the same layout).
|
|
"""
|
|
if kv_layout is None:
|
|
kv_layout = q_layout
|
|
q = _to_bhld(q, q_layout)
|
|
k = _to_bhld(k, kv_layout)
|
|
v = _to_bhld(v, kv_layout)
|
|
k, v = _expand_kv_heads(k, v, q.size(1))
|
|
attn_mask, resolved_scale = _build_attn_mask(q, k, mask, causal_offset, scale)
|
|
out = F.scaled_dot_product_attention(
|
|
q, k, v, attn_mask=attn_mask, is_causal=False, scale=resolved_scale
|
|
)
|
|
# Restore Q's original layout
|
|
if q_layout == 1:
|
|
out = out.transpose(1, 2)
|
|
return out
|
|
|
|
|
|
def _gather_kv_from_pages(
|
|
page_table: torch.Tensor,
|
|
k_cache: torch.Tensor,
|
|
v_cache: torch.Tensor,
|
|
page_size: int,
|
|
kv_len: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Gather contiguous K/V from paged cache for torch SDPA fallback.
|
|
|
|
Shapes:
|
|
page_table : [batch, max_pages] (int64)
|
|
k_cache : [n_pages, page_size, n_kv_heads, head_dim]
|
|
v_cache : same as k_cache
|
|
Returns:
|
|
k, v : [batch, kv_len, n_kv_heads, head_dim] (b l h d)
|
|
"""
|
|
batch, max_pages = page_table.shape
|
|
_, ps, n_kv_heads, head_dim = k_cache.shape
|
|
if ps != page_size:
|
|
raise ValueError(f"k_cache page_size mismatch: {ps} vs {page_size}")
|
|
|
|
# Vectorized gather: build physical page + offset indices, then advanced-index
|
|
positions = torch.arange(kv_len, device=page_table.device)
|
|
logical_pages = positions // page_size # [kv_len]
|
|
page_offsets = positions % page_size # [kv_len]
|
|
|
|
phys_pages = page_table[:, logical_pages] # [batch, kv_len]
|
|
# k_cache[phys_pages, page_offsets] → [batch, kv_len, n_kv_heads, head_dim] (b l h d)
|
|
k = k_cache[phys_pages, page_offsets]
|
|
v = v_cache[phys_pages, page_offsets]
|
|
return k, v
|
|
|
|
|
|
def attn_decode(
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
v: torch.Tensor,
|
|
mask: torch.Tensor | None = None,
|
|
causal_offset: int = -1,
|
|
scale: float = 0.0,
|
|
layout: str = "bhld",
|
|
) -> torch.Tensor:
|
|
li = _parse_layout(layout)
|
|
if _available["attn_decode"]:
|
|
return _modules["attn_decode"].attn_decode(
|
|
q,
|
|
k,
|
|
v,
|
|
mask=mask,
|
|
causal_offset=causal_offset,
|
|
scale=scale,
|
|
layout=li,
|
|
)
|
|
return _torch_fallback(q, k, v, mask, causal_offset, scale, q_layout=li)
|
|
|
|
|
|
def attn_prefill(
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
v: torch.Tensor,
|
|
mask: torch.Tensor | None = None,
|
|
causal_offset: int = -1,
|
|
scale: float = 0.0,
|
|
layout: str = "bhld",
|
|
) -> torch.Tensor:
|
|
li = _parse_layout(layout)
|
|
if _available["attn_prefill"]:
|
|
return _modules["attn_prefill"].attn_prefill(
|
|
q,
|
|
k,
|
|
v,
|
|
mask=mask,
|
|
causal_offset=causal_offset,
|
|
scale=scale,
|
|
layout=li,
|
|
)
|
|
return _torch_fallback(q, k, v, mask, causal_offset, scale, q_layout=li)
|
|
|
|
|
|
def attn_paged_decode(
|
|
q: torch.Tensor,
|
|
page_table: torch.Tensor,
|
|
k_cache: torch.Tensor,
|
|
v_cache: torch.Tensor,
|
|
page_size: int,
|
|
kv_len: int,
|
|
mask: torch.Tensor | None = None,
|
|
causal_offset: int = -1,
|
|
scale: float = 0.0,
|
|
layout: str = "bhld",
|
|
) -> torch.Tensor:
|
|
li = _parse_layout(layout)
|
|
if _available["attn_paged_decode"]:
|
|
return _modules["attn_paged_decode"].attn_paged_decode(
|
|
q,
|
|
page_table,
|
|
k_cache,
|
|
v_cache,
|
|
page_size,
|
|
kv_len,
|
|
mask=mask,
|
|
causal_offset=causal_offset,
|
|
scale=scale,
|
|
layout=li,
|
|
)
|
|
# Gathered K/V are always b l h d
|
|
k, v = _gather_kv_from_pages(page_table, k_cache, v_cache, page_size, kv_len)
|
|
return _torch_fallback(
|
|
q, k, v, mask, causal_offset, scale, q_layout=li, kv_layout=1
|
|
)
|