refactor: unify operator selection behind generic dispatch

Add astrai/extension/dispatch.py: per-family decision tables over
composable Specs, with explicit-strict / implicit-loose resolution,
ASTR_OPS env overrides, profile presets, and explain traces. Migrate
attention (behavior-preserving facade) and rotary onto it; new tests
cover spec algebra, resolution semantics, and spec-vs-supports_call
consistency.
This commit is contained in:
2026-09-01 16:35:02 +08:00
parent aabf366633
commit acbe57a0f9
5 changed files with 922 additions and 94 deletions
+44 -18
View File
@@ -1,7 +1,9 @@
"""Rotary embedding with auto-dispatch to CUDA kernel.
"""Rotary embedding dispatch (family "rotary").
Single entry point ``apply_rotary_emb(x, freqs_cis)`` — uses the fused
CUDA kernel when available, falls back to torch complex multiply otherwise.
Registered rows: the fused CUDA kernel (bf16 CUDA, inference-only) and the
torch complex-multiply fallback (autograd-safe). Selection runs through
the generic dispatcher, so ``op_backend(rotary=...)`` and
``ASTR_OPS=rotary=torch`` work exactly like for attention.
Layout: x is [batch, seq_len, n_heads, head_dim] (bf16).
freqs_cis is [batch, seq_len, dim/2, 2] (f32) — [cos, sin] pairs.
@@ -10,16 +12,17 @@ freqs_cis is [batch, seq_len, dim/2, 2] (f32) — [cos, sin] pairs.
import torch
from torch import Tensor
from astrai.extension.dispatch import (
CallContext,
ImplRecord,
Spec,
register_family,
resolve,
)
from astrai.extension.loader import is_available
from astrai.extension.ops.rotary import rotary_emb as _cuda_rotary
_cache = {"available": None}
def _cuda_available() -> bool:
if _cache["available"] is None:
_cache["available"] = is_available("rotary_emb")
return _cache["available"]
_SPEC_CUDA = Spec.cuda_device() & Spec.dtype_in(torch.bfloat16) & Spec.no_grad()
def _torch_apply(x: Tensor, freqs_cis: Tensor) -> Tensor:
@@ -33,6 +36,29 @@ def _torch_apply(x: Tensor, freqs_cis: Tensor) -> Tensor:
return x_out.to(dtype)
def _rotary_records() -> list:
return [
ImplRecord(
family="rotary",
name="cuda",
obj=_cuda_rotary,
spec=_SPEC_CUDA,
available=lambda: is_available("rotary_emb"),
priority=0,
),
ImplRecord(
family="rotary",
name="torch",
obj=_torch_apply,
spec=Spec.always(),
priority=99,
),
]
register_family("rotary", _rotary_records, lambda: _rotary_records()[-1])
def apply_rotary_emb(x: Tensor, freqs_cis: Tensor) -> Tensor:
"""Apply rotary embedding to x.
@@ -43,11 +69,11 @@ def apply_rotary_emb(x: Tensor, freqs_cis: Tensor) -> Tensor:
Returns:
[batch, seq_len, n_heads, head_dim] (bf16)
"""
if (
_cuda_available()
and not torch.is_grad_enabled()
and x.is_cuda
and x.dtype == torch.bfloat16
):
return _cuda_rotary(x, freqs_cis)
return _torch_apply(x, freqs_cis)
ctx = CallContext(
family="rotary",
dtype=x.dtype,
device_cuda=x.is_cuda,
grad_enabled=torch.is_grad_enabled(),
raw=(x, freqs_cis),
)
return resolve("rotary", ctx).record.obj(x, freqs_cis)