refactor : 将 config 对象直接传给 DecoderBlock,替代 16 个独立参数
- DecoderBlock.__init__ 改为 (config, layer_id),内部用 asdict 展开字段给 AttnFactory/FFNFactory,factory 按 __init__ 签名自动过滤 - EncoderConfig 补充 attn_type 和 ffn_type 字段 - 314 个测试全部通过
This commit is contained in:
+1
-13
@@ -26,19 +26,7 @@ class EmbeddingEncoder(AutoModel):
|
||||
self.embed_tokens = Embedding(config.vocab_size, config.dim)
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DecoderBlock(
|
||||
config.dim,
|
||||
config.n_heads,
|
||||
config.dim_ffn,
|
||||
config.n_kv_heads,
|
||||
config.norm_eps,
|
||||
config.use_qk_norm,
|
||||
config.use_gated_attention,
|
||||
layer_id,
|
||||
)
|
||||
for layer_id in range(config.n_layers)
|
||||
]
|
||||
[DecoderBlock(config, layer_id) for layer_id in range(config.n_layers)]
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.dim, config.norm_eps)
|
||||
|
||||
Reference in New Issue
Block a user