feat : GPT-2 residual scaling weight init

- Linear: normal(0, init_std) replaces kaiming_uniform_(a=sqrt(5))
- o_proj / mlp.down: init_std = 0.02 / sqrt(2 * n_layers)
- MoE: expert down scaled by 1/sqrt(1/n_shared + 1/K)
- Embedding: normal(0, 0.02), unchanged

astrai/model/components/linear.py

@@ -5,13 +5,16 @@ from torch import Tensor
 
 
 class Linear(nn.Module):
-    def __init__(self, in_dim: int, out_dim: int, bias: bool = False):
+    def __init__(
+        self, in_dim: int, out_dim: int, bias: bool = False, init_std: float = 0.02
+    ):
         super().__init__()
         self.weight = nn.Parameter(torch.empty((out_dim, in_dim)))
         self.bias = nn.Parameter(torch.zeros(out_dim)) if bias else None
+        self.init_std = init_std
 
     def reset_parameters(self):
-        nn.init.kaiming_uniform_(self.weight, a=5**0.5)
+        nn.init.normal_(self.weight, mean=0.0, std=self.init_std)
         if self.bias is not None:
             fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight)
             bound = 1 / (fan_in**0.5)