Compare commits
4
Commits
a7bbc7b29f
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e6be33aa53 | ||
|
|
0a1d0573ae | ||
|
|
0c2bc916f2 | ||
|
|
4e70e827ff |
@@ -154,8 +154,16 @@ pipeline proceeds as follows:
|
||||
previously kept sample $\mathbf{s}'$.
|
||||
\end{enumerate}
|
||||
|
||||
An optional LLM-as-Judge scoring module provides multi-dimensional
|
||||
quality scores that can be used to filter low-quality samples.
|
||||
A length filter is applied to SFT samples based on the IFD
|
||||
length-bias analysis in Appendix~\ref{sec:ifd_bias}
|
||||
(Figure~\ref{fig:length_bias}): instruction--response pairs
|
||||
whose response contains fewer than 15 tokens are discarded,
|
||||
because short replies exhibit both high per-token perplexity
|
||||
($L_{\text{uncond}} \approx 6\text{--}8$, PPL~$\approx 400\text{--}3000$)
|
||||
and wide variance in both $L_{\text{cond}}$ and $L_{\text{uncond}}$,
|
||||
which would distort downstream IFD-based difficulty estimates.
|
||||
The threshold is applied per-field, analogous to the pretraining
|
||||
filter described above.
|
||||
|
||||
\subsection{DPO Data Generation}
|
||||
|
||||
@@ -302,7 +310,7 @@ Sequence length & 2,048 tokens \\
|
||||
\centering
|
||||
\includegraphics[width=0.50\linewidth]{data/loss_compare.png}
|
||||
\caption{Training loss curves: GPT-2 residual scaling vs.~Kaiming
|
||||
initialization over $\sim$20B tokens.}
|
||||
initialization over $\sim$5B tokens.}
|
||||
\label{fig:loss}
|
||||
\end{figure}
|
||||
|
||||
@@ -547,17 +555,6 @@ spread.}
|
||||
\label{fig:ckpt_weight_density}
|
||||
\end{figure}
|
||||
|
||||
\begin{figure}[H]
|
||||
\centering
|
||||
\includegraphics[width=0.95\linewidth]{data/ckpt_weight_density_per_run.png}
|
||||
\caption{Per-checkpoint weight density breakdowns. The three checkpoints
|
||||
are \texttt{norm-15bt} (AdamW, Normal init, 15B tokens), \texttt{kami-15bt}
|
||||
(AdamW, GPT-2 residual scaling, 15B tokens), and \texttt{muon-25bt}
|
||||
(Muon, 25B tokens). Note: the iteration labels inside the figure reflect
|
||||
legacy script metadata and should be ignored in favour of the token
|
||||
budgets stated here.}
|
||||
\label{fig:ckpt_weight_density_per_run}
|
||||
\end{figure}
|
||||
|
||||
% ======================================================================
|
||||
\section{Conclusion}
|
||||
|
||||
Reference in New Issue
Block a user