fix: add caption package, fix overfull hbox, correct RoFormer author names

This commit is contained in:
ViperEkura 2026-06-25 16:53:26 +08:00
parent 6f9278b35b
commit 3db1096c78
1 changed files with 7 additions and 4 deletions

View File

@ -9,6 +9,7 @@
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{float}
\usepackage{caption}
\usepackage{enumitem}
\usepackage{url}
@ -101,6 +102,7 @@ with Rotary Position Embedding (RoPE)~\cite{su2024roformer}.
\begin{table}[H]
\centering
\caption{Model configuration. Total: $\sim$1.2B parameters.}
\label{tab:model_config}
\begin{tabular}{@{}lrlr@{}}
\toprule
\textbf{Parameter} & \textbf{Value} & \textbf{Parameter} & \textbf{Value} \\
@ -235,9 +237,10 @@ Without the $1/\sqrt{2L}$ factor on $\mathbf{W}_o$ and
$\mathbf{W}_{\text{down}}$, the per-block residual variance becomes $0.689$
instead of $0.689/L \approx 0.014$. After $L=24$ blocks:
\begin{equation}
\text{Without scaling: } \Var(\mathbf{x}_{24}) \approx 1 + 24 \times 0.689 = 17.5,
\qquad
\text{With scaling: } \Var(\mathbf{x}_{24}) \approx 1 + 24 \times 0.014 = 1.34.
\begin{aligned}
\text{Without scaling: } \Var(\mathbf{x}_{24}) &\approx 1 + 24 \times 0.689 = 17.5,\\
\text{With scaling: } \Var(\mathbf{x}_{24}) &\approx 1 + 24 \times 0.014 = 1.34.
\end{aligned}
\end{equation}
\textbf{Stage 2: Saturation.} The 17.5$\times$ inflated variance at deep
@ -314,7 +317,7 @@ N.~Shazeer. GLU variants improve Transformer.
\textit{arXiv:2002.05202}, 2020.
\bibitem{su2024roformer}
J.~Su, M.~Ahmed, Y.~Lu, S.~Pan, W.~Bo, Y.~Liu.
J.~Su, A.~Murtadha, Y.~Lu, S.~Pan, B.~Wen, Y.~Liu.
Roformer: Enhanced transformer with rotary position embedding.
\textit{Neurocomputing}, 568:127063, 2024.