refactor : align config field names with Hugging Face

- dim -> hidden_size, n_layers -> num_hidden_layers
- dim_ffn -> intermediate_size, n_heads -> num_attention_heads
- n_kv_heads -> num_key_value_heads, max_len -> max_position_embeddings
- norm_eps -> rms_norm_eps, tie_weight -> tie_word_embeddings
- update model, inference, training, scripts, tests, docs
This commit is contained in:
2026-07-20 22:05:31 +08:00
parent d7ac66fb73
commit 0c86c89af4
23 changed files with 202 additions and 166 deletions
+11 -3
View File
@@ -14,10 +14,18 @@ class DecoderBlock(nn.Module):
def __init__(self, config, layer_id: int):
super().__init__()
cfg = asdict(config)
cfg["down_init_std"] = 0.02 / (2 * config.n_layers) ** 0.5
cfg.update(
dim=config.hidden_size,
dim_ffn=config.intermediate_size,
n_layers=config.num_hidden_layers,
n_heads=config.num_attention_heads,
n_kv_heads=config.num_key_value_heads,
norm_eps=config.rms_norm_eps,
down_init_std=0.02 / (2 * config.num_hidden_layers) ** 0.5,
)
self.attention = AttnFactory.create(config.attn_type, **cfg, layer_id=layer_id)
self.input_norm = RMSNorm(config.dim, config.norm_eps)
self.post_attention_norm = RMSNorm(config.dim, config.norm_eps)
self.input_norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
self.post_attention_norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
self.mlp = FFNFactory.create(config.ffn_type, **cfg)
def forward(