refactor: unify extension API to blhd layout and is_causal
- Rename ops.py to attention_ops.py - Remove layout/scale params: fixed blhd, auto scale - Replace causal_offset with is_causal bool - Move SDPA fallback to backend, ops only calls CUDA kernels - Update __init__.py exports
This commit is contained in:
@@ -4,27 +4,39 @@ Public API:
|
||||
- ``attn_decode`` — single-query decode attention
|
||||
- ``attn_prefill`` — multi-query prefill attention
|
||||
- ``attn_paged_decode`` — paged decode attention (direct page-table access)
|
||||
- ``AttentionBackend`` — ABC for attention computation strategies
|
||||
- ``TorchNativeBackend`` — default SDPA backend with KV cache I/O
|
||||
|
||||
Interface (shared by all wrappers):
|
||||
causal_offset: -1 = non-causal; >=0 = absolute position of first Q token
|
||||
mask: 2D [batch, kv_len] or 3D [batch, q_len, kv_len] (bool, True = keep)
|
||||
scale: 0.0 = auto (1/sqrt(head_dim)); >0 = explicit
|
||||
layout: "bhld" (default) or "blhd"
|
||||
|
||||
Causal and mask can coexist — both are applied simultaneously.
|
||||
Layout convention: all q/k/v are ``[batch, seq_len, n_heads, head_dim]``
|
||||
(blhd). Scale is always ``1/sqrt(head_dim)``.
|
||||
|
||||
Each wrapper dispatches to its compiled CUDA kernel (``astrai.extension.attn_*``)
|
||||
when available, otherwise falls back to ``torch.nn.functional.scaled_dot_product_attention``.
|
||||
"""
|
||||
|
||||
from astrai.extension.attention_backend import (
|
||||
ATTN_BACKEND,
|
||||
AttentionBackend,
|
||||
TorchNativeBackend,
|
||||
attn_backend,
|
||||
get_backend,
|
||||
)
|
||||
from astrai.extension.attention_ops import (
|
||||
attn_decode,
|
||||
attn_paged_decode,
|
||||
attn_prefill,
|
||||
)
|
||||
from astrai.extension.loader import KERNEL_NAMES, is_available
|
||||
from astrai.extension.ops import attention, attn_decode, attn_paged_decode, attn_prefill
|
||||
|
||||
__all__ = [
|
||||
"ATTN_BACKEND",
|
||||
"AttentionBackend",
|
||||
"TorchNativeBackend",
|
||||
"attn_backend",
|
||||
"get_backend",
|
||||
"attn_decode",
|
||||
"attn_paged_decode",
|
||||
"attn_prefill",
|
||||
"attention",
|
||||
"is_available",
|
||||
"KERNEL_NAMES",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user