26 lines
679 B
Python
26 lines
679 B
Python
from astrai.extension.rotary_backend import apply_rotary_emb
|
|
from astrai.model.components.attention import GQA, MLA
|
|
from astrai.model.components.decoder_block import DecoderBlock
|
|
from astrai.model.components.embedding import Embedding
|
|
from astrai.model.components.linear import Linear
|
|
from astrai.model.components.mlp import MLP, DeepSeekMoE
|
|
from astrai.model.components.norm import RMSNorm
|
|
from astrai.model.components.rope import (
|
|
RotaryEmbedding,
|
|
get_rotary_emb,
|
|
)
|
|
|
|
__all__ = [
|
|
"Linear",
|
|
"RMSNorm",
|
|
"MLP",
|
|
"DeepSeekMoE",
|
|
"Embedding",
|
|
"GQA",
|
|
"MLA",
|
|
"DecoderBlock",
|
|
"RotaryEmbedding",
|
|
"apply_rotary_emb",
|
|
"get_rotary_emb",
|
|
]
|