feat: support HEAD_DIM=32 and split extension into loader/ops

- add case 32 to decode/prefill dispatch switch
- fix swiz_col out-of-bounds for HEAD_DIM=32: XOR mask now limited to chunk count (3 for 32, 7 for >=64) instead of always 7, which produced column offsets >= LD=32 and corrupted shared memory
- restructure decode dispatch to #ifndef/#else/#endif matching prefill
- split astrai/extension/__init__.py into loader.py (kernel .so discovery) and ops.py (wrapper functions + torch SDPA fallback); __init__.py now re-exports the public API
This commit is contained in:
2026-07-08 14:14:11 +08:00
parent 9ebaea840f
commit fd65b9bc23
7 changed files with 168 additions and 102 deletions
+36
View File
@@ -0,0 +1,36 @@
"""Dynamic discovery and loading of compiled CUDA kernel modules.
Each kernel is registered in ``csrc/build.py`` and built into a ``.so`` placed
in this package directory. On import we try to load each one; kernels that
failed to build (or are running on a CPU-only machine) are marked unavailable
so the wrapper functions can fall back to ``torch`` SDPA.
"""
import importlib
import logging
logger = logging.getLogger(__name__)
KERNEL_NAMES = ["gqa_decode_attn", "gqa_prefill_attn"]
_available: dict[str, bool] = {}
_modules: dict[str, object] = {}
for _name in KERNEL_NAMES:
try:
_mod = importlib.import_module(f".{_name}", package=__package__)
_available[_name] = True
_modules[_name] = _mod
except ImportError:
_available[_name] = False
_modules[_name] = None
def is_available(name: str) -> bool:
"""Return ``True`` if the compiled kernel ``name`` was loaded."""
return _available.get(name, False)
def get_module(name: str) -> object:
"""Return the loaded kernel module for ``name``, or ``None`` if unavailable."""
return _modules.get(name)