ViperEkura/AstrAI
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor : align config field names with Hugging Face

Browse Source 
- dim -> hidden_size, n_layers -> num_hidden_layers
- dim_ffn -> intermediate_size, n_heads -> num_attention_heads
- n_kv_heads -> num_key_value_heads, max_len -> max_position_embeddings
- norm_eps -> rms_norm_eps, tie_weight -> tie_word_embeddings
- update model, inference, training, scripts, tests, docs
This commit is contained in:
ViperEkura
2026-07-20 22:05:31 +08:00
parent d7ac66fb73
commit 0c86c89af4
23 changed files with 202 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
astrai/config/model_config.py
+15 -15
View File
@@ -29,19 +29,19 @@ class AutoRegressiveLMConfig(BaseModelConfig):
"""Configuration for autoregressive language model."""
vocab_size: Optional[int] = None
dim: Optional[int] = None
n_layers: Optional[int] = None
norm_eps: Optional[float] = None
dim_ffn: Optional[int] = None
tie_weight: Optional[bool] = None
hidden_size: Optional[int] = None
num_hidden_layers: Optional[int] = None
rms_norm_eps: Optional[float] = None
intermediate_size: Optional[int] = None
tie_word_embeddings: Optional[bool] = None
max_len: Optional[int] = None
max_position_embeddings: Optional[int] = None
rope_theta: Optional[float] = None
rope_scaling: Optional[dict] = None
attn_type: str = "gqa"
n_heads: Optional[int] = None
n_kv_heads: Optional[int] = None
num_attention_heads: Optional[int] = None
num_key_value_heads: Optional[int] = None
use_qk_norm: Optional[bool] = None
use_gated_attention: Optional[bool] = None
@@ -62,18 +62,18 @@ class EncoderConfig(BaseModelConfig):
"""Configuration for embedding encoder model."""
vocab_size: Optional[int] = None
dim: Optional[int] = None
n_layers: Optional[int] = None
norm_eps: Optional[float] = None
dim_ffn: Optional[int] = None
hidden_size: Optional[int] = None
num_hidden_layers: Optional[int] = None
rms_norm_eps: Optional[float] = None
intermediate_size: Optional[int] = None
max_len: Optional[int] = None
max_position_embeddings: Optional[int] = None
rope_theta: Optional[float] = None
rope_scaling: Optional[dict] = None
attn_type: str = "gqa"
n_heads: Optional[int] = None
n_kv_heads: Optional[int] = None
num_attention_heads: Optional[int] = None
num_key_value_heads: Optional[int] = None
use_qk_norm: Optional[bool] = None
use_gated_attention: Optional[bool] = None