perf: accelerate decode linear with bf16 gemv
- add decode-shape benchmark harness - add bf16 GEMV CUDA primitive with head-dim generic kernel - dispatch decode-time linear layers to gemv for M=1 - extend gemv coverage to small decode batches
This commit is contained in:
@@ -11,6 +11,7 @@ from astrai.extension.backend.attention import (
|
||||
attn_backend,
|
||||
get_backend,
|
||||
)
|
||||
from astrai.extension.backend.linear import linear
|
||||
from astrai.extension.backend.rotary import apply_rotary_emb
|
||||
|
||||
__all__ = [
|
||||
@@ -24,4 +25,5 @@ __all__ = [
|
||||
"attention",
|
||||
"attn_backend",
|
||||
"get_backend",
|
||||
"linear",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user