ViperEkura
88ec63121d
- Linear: normal(0, init_std) replaces kaiming_uniform_(a=sqrt(5)) - o_proj / mlp.down: init_std = 0.02 / sqrt(2 * n_layers) - MoE: expert down scaled by 1/sqrt(1/n_shared + 1/K) - Embedding: normal(0, 0.02), unchanged
25 lines
825 B
Python
25 lines
825 B
Python
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
from torch import Tensor
|
|
|
|
|
|
class Linear(nn.Module):
|
|
def __init__(
|
|
self, in_dim: int, out_dim: int, bias: bool = False, init_std: float = 0.02
|
|
):
|
|
super().__init__()
|
|
self.weight = nn.Parameter(torch.empty((out_dim, in_dim)))
|
|
self.bias = nn.Parameter(torch.zeros(out_dim)) if bias else None
|
|
self.init_std = init_std
|
|
|
|
def reset_parameters(self):
|
|
nn.init.normal_(self.weight, mean=0.0, std=self.init_std)
|
|
if self.bias is not None:
|
|
fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight)
|
|
bound = 1 / (fan_in**0.5)
|
|
nn.init.uniform_(self.bias, -bound, bound)
|
|
|
|
def forward(self, x: Tensor) -> Tensor:
|
|
return F.linear(x, self.weight, self.bias)
|