refactor: unify operator selection behind generic dispatch
Add astrai/extension/dispatch.py: per-family decision tables over composable Specs, with explicit-strict / implicit-loose resolution, ASTR_OPS env overrides, profile presets, and explain traces. Migrate attention (behavior-preserving facade) and rotary onto it; new tests cover spec algebra, resolution semantics, and spec-vs-supports_call consistency.
This commit is contained in:
@@ -1,7 +1,9 @@
|
||||
"""Rotary embedding with auto-dispatch to CUDA kernel.
|
||||
"""Rotary embedding dispatch (family "rotary").
|
||||
|
||||
Single entry point ``apply_rotary_emb(x, freqs_cis)`` — uses the fused
|
||||
CUDA kernel when available, falls back to torch complex multiply otherwise.
|
||||
Registered rows: the fused CUDA kernel (bf16 CUDA, inference-only) and the
|
||||
torch complex-multiply fallback (autograd-safe). Selection runs through
|
||||
the generic dispatcher, so ``op_backend(rotary=...)`` and
|
||||
``ASTR_OPS=rotary=torch`` work exactly like for attention.
|
||||
|
||||
Layout: x is [batch, seq_len, n_heads, head_dim] (bf16).
|
||||
freqs_cis is [batch, seq_len, dim/2, 2] (f32) — [cos, sin] pairs.
|
||||
@@ -10,16 +12,17 @@ freqs_cis is [batch, seq_len, dim/2, 2] (f32) — [cos, sin] pairs.
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from astrai.extension.dispatch import (
|
||||
CallContext,
|
||||
ImplRecord,
|
||||
Spec,
|
||||
register_family,
|
||||
resolve,
|
||||
)
|
||||
from astrai.extension.loader import is_available
|
||||
from astrai.extension.ops.rotary import rotary_emb as _cuda_rotary
|
||||
|
||||
_cache = {"available": None}
|
||||
|
||||
|
||||
def _cuda_available() -> bool:
|
||||
if _cache["available"] is None:
|
||||
_cache["available"] = is_available("rotary_emb")
|
||||
return _cache["available"]
|
||||
_SPEC_CUDA = Spec.cuda_device() & Spec.dtype_in(torch.bfloat16) & Spec.no_grad()
|
||||
|
||||
|
||||
def _torch_apply(x: Tensor, freqs_cis: Tensor) -> Tensor:
|
||||
@@ -33,6 +36,29 @@ def _torch_apply(x: Tensor, freqs_cis: Tensor) -> Tensor:
|
||||
return x_out.to(dtype)
|
||||
|
||||
|
||||
def _rotary_records() -> list:
|
||||
return [
|
||||
ImplRecord(
|
||||
family="rotary",
|
||||
name="cuda",
|
||||
obj=_cuda_rotary,
|
||||
spec=_SPEC_CUDA,
|
||||
available=lambda: is_available("rotary_emb"),
|
||||
priority=0,
|
||||
),
|
||||
ImplRecord(
|
||||
family="rotary",
|
||||
name="torch",
|
||||
obj=_torch_apply,
|
||||
spec=Spec.always(),
|
||||
priority=99,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
register_family("rotary", _rotary_records, lambda: _rotary_records()[-1])
|
||||
|
||||
|
||||
def apply_rotary_emb(x: Tensor, freqs_cis: Tensor) -> Tensor:
|
||||
"""Apply rotary embedding to x.
|
||||
|
||||
@@ -43,11 +69,11 @@ def apply_rotary_emb(x: Tensor, freqs_cis: Tensor) -> Tensor:
|
||||
Returns:
|
||||
[batch, seq_len, n_heads, head_dim] (bf16)
|
||||
"""
|
||||
if (
|
||||
_cuda_available()
|
||||
and not torch.is_grad_enabled()
|
||||
and x.is_cuda
|
||||
and x.dtype == torch.bfloat16
|
||||
):
|
||||
return _cuda_rotary(x, freqs_cis)
|
||||
return _torch_apply(x, freqs_cis)
|
||||
ctx = CallContext(
|
||||
family="rotary",
|
||||
dtype=x.dtype,
|
||||
device_cuda=x.is_cuda,
|
||||
grad_enabled=torch.is_grad_enabled(),
|
||||
raw=(x, freqs_cis),
|
||||
)
|
||||
return resolve("rotary", ctx).record.obj(x, freqs_cis)
|
||||
|
||||
Reference in New Issue
Block a user