AstrAI-paper/main.tex

\documentclass[11pt,a4paper]{article}

% ===== Packages =====
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{newtxtext,newtxmath}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath}
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{float}
\usepackage{caption}
\usepackage{enumitem}
\usepackage{url}
\usepackage{microtype}

\DeclareMathOperator{\Var}{Var}

\title{End-to-End Training of a 1.2B Transformer with AstrAI \\
       \large Data Pipeline, Distributed Training, and Ablations on Optimizer, Initialization, and BF16 Numerical Stability}

\author{AstrAI Contributors}
\date{}

\begin{document}

\maketitle

\begin{abstract}
We present {\sc AstrAI}, an open-source framework for end-to-end training
of a 1.2B-parameter Transformer on $\sim$20B tokens. The pipeline covers
JSON-driven BBPE preprocessing with multi-strategy packing, HDF5/mmap
storage backends, and a companion SFT pipeline ({\sc Alembic}) with MinHash
deduplication and LLM-as-Judge scoring. The 24-layer decoder uses GQA,
SwiGLU, RoPE, and RMSNorm, trained with a hybrid Muon/AdamW optimizer and
WSD (Warmup--Stable--Decay) scheduling under DDP/FSDP. A BF16 stability analysis shows that
GPT-2 residual scaling substantially reduces per-block residual variance
accumulation, keeping post-training variance well below the overflow
threshold of standard initialization; empirically this yields a sustained
loss advantage over Kaiming initialization throughout training.
Systematic ablations show that the hybrid Muon/AdamW optimizer
outperforms pure AdamW on 2D weight matrices, and that GPT-2 residual
scaling yields a sustained loss advantage over both Kaiming and Normal
initialization throughout training. Post-training weight distribution
analysis across three checkpoints---varying optimizer, initialization,
and training budget---confirms that residual-scaled projections
maintain narrow distributions throughout training, preserving the
numerical stability established at initialization. An SVD-based
effective rank analysis further reveals that the model operates near
its representational capacity, with attention Q/O projections
consistently showing lower utilization than K/V projections, a pattern
stable across all configurations and consistent with the low-rank
structure induced by grouped query attention.
\end{abstract}

% ======================================================================
\section{Introduction}
% ======================================================================

Training a billion-parameter language model end-to-end involves far more than
model architecture. Data must be preprocessed and stored efficiently, the
training loop must handle distributed parallelism, gradient accumulation,
checkpointing, and logging---and numerical pitfalls must be diagnosed and
fixed. This paper describes the complete workflow using {\sc AstrAI}~\cite{astrai}, an
open-source framework for Transformer training and inference. Beyond the
pipeline, we conduct systematic ablations on optimizers (hybrid Muon/AdamW
versus pure AdamW) and initializations (GPT-2 residual scaling, Kaiming,
and Normal), and analyse a BF16 precision issue encountered along the way.

% ======================================================================
\section{Data Pipeline}
% ======================================================================

\subsection{Preprocessing}

Raw data arrives as JSONL files. The preprocessing pipeline is configured
via a JSON specification that defines:

\begin{itemize}[nosep]
  \item \textbf{Tokenization}: BBPE tokenizer (100K vocabulary) with standard
        special tokens.
  \item \textbf{Masking}: Declarative loss mask assignment per section
        (e.g.,~mask user input, compute loss on assistant response).
  \item \textbf{Packing}: Documents concatenated via \texttt{simple}
        (sequential), \texttt{bfd} (best-fit decreasing), or
        \texttt{bfd\_\allowbreak{}split} strategies.
  \item \textbf{Position IDs}: \texttt{none}, \texttt{doc\_reset} (per-document
        boundary), or \texttt{continuous}.
  \item \textbf{Output}: Tokenized sequences written to \texttt{.h5} or
        \texttt{.bin} shards, auto-split at 100M tokens per shard.
\end{itemize}

Samples shorter than 50~chars or longer than 2M~chars are filtered out.

\subsection{Storage Backends}

Two storage backends serve the DataLoader:

\begin{itemize}[nosep]
  \item \textbf{H5Store}: HDF5-based, memory-loaded with shared-memory support
        for multi-worker access.
  \item \textbf{MmapStore}: Zero-copy memory-mapped \texttt{.bin} files shared
        via OS page cache.
\end{itemize}

A resumable distributed sampler provides seed-based shuffle with
epoch/iteration resume.

\subsection{SFT Data Cleaning}

For supervised fine-tuning (SFT), raw data requires additional curation
beyond pretraining tokenization. {\sc Alembic}~\cite{alembic} is a companion
pipeline that handles SFT data generation, cleaning, and quality scoring:
three-generation strategies (topic-driven, seed-driven, self-instruct),
built-in cleaning (HTML/URL/markdown removal, char/word repetition filters),
and a MinHash-based near-duplicate detection system~\cite{broder1997syntactic}.
Given a set of $P$ hash functions ($P=128$) and a text $T$, the MinHash
pipeline proceeds as follows:

\begin{enumerate}[nosep,leftmargin=*]
  \item \textbf{Tokenization}: $T$ is split into character $n$-grams
    ($n=3$):
    \begin{equation}
      \Gamma(T) = \{\,c_i c_{i+1} c_{i+2} \mid i = 1,\dots,|T|-2 \,\}.
    \end{equation}
  \item \textbf{Signature}: For each hash function $h_k$, the minimum hash
    value over all $n$-grams forms the $k$-th element of the fingerprint:
    \begin{equation}
      s_k = \min_{t \in \Gamma(T)} h_k(t), \qquad
      h_k(t) = \operatorname{SHA256}(42 : k : t)_{[0:63]}.
    \end{equation}
    The full fingerprint is $\mathbf{s} = (s_1,\dots,s_P)$.
  \item \textbf{Similarity}: The Jaccard similarity between two sets is
    estimated by the fraction of agreeing fingerprint positions:
    \begin{equation}
      \widehat{J}(\mathbf{s}^{(a)},\mathbf{s}^{(b)}) =
      \frac{|\{\,k \mid s^{(a)}_k = s^{(b)}_k \,\}|}{P}.
    \end{equation}
  \item \textbf{Filtering}: Samples are processed sequentially; a sample
    is dropped if $\widehat{J}(\mathbf{s}, \mathbf{s}') \ge 0.7$ for any
    previously kept sample $\mathbf{s}'$.
\end{enumerate}

An optional LLM-as-Judge scoring module provides multi-dimensional
quality scores that can be used to filter low-quality samples.

\subsection{DPO Data Generation}

To construct pairwise preference data for Direct Preference
Optimization, we start from a bilingual (Chinese--English) instruction
set curated by the same MinHash pipeline described above.  For each
prompt $x$ in the instruction set, we generate two responses:

\begin{itemize}[nosep]
  \item \textbf{Chosen} $y_w$: generated by the reference model
    $\pi_{\text{ref}}$, i.e.~the SFT checkpoint after supervised
    fine-tuning on the curated instruction data.
  \item \textbf{Rejected} $y_l$: generated by the base model
    $\pi_{\text{base}}$, i.e.~the model at the end of pretraining
    before any instruction tuning.
\end{itemize}

Both generations use greedy decoding to eliminate sampling variance and
ensure that the preference signal reflects model capability rather than
decoding randomness.  The resulting preference pairs
$(x, y_w, y_l)$ are stored in the same JSONL format used for SFT and
fed directly into the DPO training loop
(Section~\ref{sec:dpo}).  Because the base model and the reference
model share the same architecture but differ in instruction-following
ability, the contrast between $y_w$ and $y_l$ is sharp and consistent,
which stabilises the DPO gradient updates.  All instructions are
balanced across Chinese and English domains to preserve bilingual
alignment capability during preference optimisation.
\section{Model Architecture}
% ======================================================================

The model is a 24-layer decoder-only Transformer with Grouped Query Attention
(GQA)~\cite{ainslie2023gqa}, SwiGLU feed-forward blocks~\cite{shazeer2020glu},
and Rotary Position Embedding (RoPE)~\cite{su2024roformer}.
Table~\ref{tab:model_config} summarizes the configuration.

\begin{table}[H]
\centering
\caption{Model configuration. Total: $\sim$1.2B parameters.}
\label{tab:model_config}
\begin{tabular}{@{}lrlr@{}}
\toprule
\textbf{Parameter} & \textbf{Value} & \textbf{Parameter} & \textbf{Value} \\
\midrule
Vocabulary ($V$) & 100,000 & Hidden dim ($d$) & 1,536 \\
Layers ($L$) & 24 & FFN dim ($d_{\textit{ffn}}$) & 6,912 \\
Query heads & 24 & KV heads & 4 \\
Head dim & 64 & Max length & 2,048 \\
Norm & RMSNorm ($\epsilon=10^{-5}$) & RoPE $\theta$ & 10,000 \\
\bottomrule
\end{tabular}
\end{table}

With Grouped Query Attention~\cite{ainslie2023gqa} ($n_q = 24$ query
heads, $n_{kv} = 4$ key/value heads, group size $g = n_q / n_{kv} = 6$):
\begin{equation}
\begin{aligned}
\operatorname{GQA}(\mathbf{X}) &= \operatorname{Concat}\bigl(\operatorname{head}_1,\dots,\operatorname{head}_{n_q}\bigr)\mathbf{W}_O,\\[2mm]
\operatorname{head}_i &= \operatorname{Attn}\Bigl(
\mathbf{X}\mathbf{W}_Q^{(i)},\,
\mathbf{X}\mathbf{W}_K^{(\lfloor i / g \rfloor)},\,
\mathbf{X}\mathbf{W}_V^{(\lfloor i / g \rfloor)}
\Bigr),
\end{aligned}
\end{equation}
where $\operatorname{Attn}(\mathbf{Q},\mathbf{K},\mathbf{V}) =
\operatorname{Softmax}(\mathbf{Q}\mathbf{K}^{\mkern-1mu\mathsf{T}} / \sqrt{d_h})\mathbf{V}$.
Rotary Position Embedding (RoPE)~\cite{su2024roformer} encodes position
$m$ by rotating pairs of hidden dimensions:
\begin{equation}
\operatorname{RoPE}(\mathbf{x}_m)_i =
\begin{cases}
x_{m,i}\cos(m\theta_{j}) - x_{m,i+1}\sin(m\theta_{j}), & i = 2j,\\[2mm]
x_{m,i-1}\sin(m\theta_{j}) + x_{m,i}\cos(m\theta_{j}), & i = 2j+1,
\end{cases}
\end{equation}
with frequency $\theta_j = 10000^{-2j/d}$ for $j = 0,\dots,d/2-1$.
The SwiGLU~\cite{shazeer2020glu} feed-forward applies a gated Swish
non-linearity:
\begin{equation}
\operatorname{MLP}(\mathbf{x}) = \mathbf{W}_{\text{down}}\Bigl(
\mathbf{W}_{\text{up}}\mathbf{x} \odot
\operatorname{SiLU}\bigl(\mathbf{W}_{\text{gate}}\mathbf{x}\bigr)
\Bigr),
\end{equation}
where $\operatorname{SiLU}(z) = z / (1 + e^{-z})$.

Each decoder block $\ell$ then applies pre-norm residual connections:
\begin{equation}
\begin{aligned}
\mathbf{h}_\ell &= \mathbf{x}_\ell + \operatorname{GQA}\bigl(\operatorname{RMSNorm}(\mathbf{x}_\ell)\bigr),\\[2mm]
\mathbf{x}_{\ell+1} &= \mathbf{h}_\ell + \operatorname{MLP}\bigl(\operatorname{RMSNorm}(\mathbf{h}_\ell)\bigr).
\end{aligned}
\end{equation}

\subsection{Initialization}

Linear weights follow $\mathcal{N}(0, 0.02)$; embeddings follow
$\mathcal{N}(0, 0.02)$. The output projection $\mathbf{W}_o$ and FFN
down-projection $\mathbf{W}_{\text{down}}$ use residual-scaled
initialization~\cite{radford2019gpt2}:
\begin{equation}
\sigma_o = \sigma_{\text{down}} = 0.02 / \sqrt{2L}.
\end{equation}
This scaling is critical for BF16 stability (Section~\ref{sec:num-stability}).

% ======================================================================
\section{Training Configuration}
% ======================================================================

The model is trained on next-token cross-entropy loss:
\begin{equation}
\mathcal{L} = -\sum_{t=1}^{T} \log P(x_t \mid x_{<t}; \theta).
\end{equation}

Training uses a hybrid optimizer: Muon for 2D weight matrices and AdamW~\cite{loshchilov2019adamw} for 1D parameters (embeddings, biases, LayerNorm), with WSD (Warmup--Stable--Decay)
learning rate scheduling (2\% warmup) and global L2 gradient clipping. The framework supports DDP and FSDP for multi-GPU distribution,
with gradient accumulation to manage memory.
Table~\ref{tab:train_params} lists the key hyperparameters.

\begin{table}[H]
\centering
\caption{Training hyperparameters for the 1.2B run.}
\label{tab:train_params}
\begin{tabular}{@{}lr@{}}
\toprule
\textbf{Hyperparameter} & \textbf{Value} \\
\midrule
Precision & BF16 (weights + optimizer states) \\
  Optimizer & Hybrid Muon/AdamW$^a$, $\eta=1.5\times10^{-4}$ \\
Betas & $(0.9, 0.95)$, weight decay $0.1$ \\
Gradient clip & Global L2, max norm $1.0$ \\
  Scheduler & WSD (warmup 2\%, stable, decay) \\
Batch size & 4 per device $\times$ 4 GPUs $\times$ 32 accumulation \\
Sequence length & 2,048 tokens \\
  Total tokens & $\sim$25B ($\approx$23k steps) \\
\bottomrule
\multicolumn{2}{@{}l@{}}{\footnotesize $^a$Muon applied to 2D weight matrices (attention projections and FFN layers);}\\
\multicolumn{2}{@{}l@{}}{\footnotesize \phantom{$^a$}AdamW applied to 1D parameters (embeddings, biases, LayerNorm scales).}\\
\end{tabular}
\end{table}

\begin{figure}[H]
\centering
\includegraphics[width=0.50\linewidth]{data/loss_compare.png}
\caption{Training loss curves: GPT-2 residual scaling vs.~Kaiming
initialization over $\sim$20B tokens.}
\label{fig:loss}
\end{figure}

\begin{figure}[H]
\centering
\includegraphics[width=0.95\linewidth]{data/ckpt_comparison.png}
\caption{Optimizer and initialization comparison.}
\label{fig:ckpt_comparison}
\end{figure}

Figure~\ref{fig:ckpt_comparison} compares four configurations. The left panel shows training loss for Muon (Embedding Adam + 1D Adam), Muon (Embedding Muon + 1D Adam), Kaiming init, and Normal init; the center panel zooms in on the two current Muon variants; and the right panel shows gradient norms over optimizer steps. The older Kaiming and Normal initializations converge more slowly and plateau at higher loss. Between the current variants, using Adam for the embedding layer yields lower loss and more stable gradients than using Muon embeddings.

\begin{figure}[H]
\centering
\includegraphics[width=0.85\linewidth]{data/muon_pt.png}
\caption{Muon optimizer training dynamics: training loss and learning
rate schedule for the hybrid Muon/AdamW configuration across $\sim$25B
tokens. The Muon optimizer is applied to all 2D weight matrices
(attention projections and FFN layers), while AdamW handles 1D
parameters (embeddings, biases, and normalization scales). The WSD
schedule (warmup, stable phase, and final decay) is visible in the lower panel.}
\label{fig:muon_pt}
\end{figure}

Figure~\ref{fig:muon_pt} shows the extended training dynamics of the
Muon optimizer configuration over $\sim$25B tokens. The loss curve
exhibits the expected power-law decay in the early phase (0--5B tokens),
followed by a gradual plateau as the model approaches convergence on
the pretraining distribution. The WSD schedule holds the learning rate
constant during the long stable phase and then decays at the end of
training, ensuring stable weight updates in the final phase.

\begin{figure}[H]
\centering
\includegraphics[width=0.85\linewidth]{data/sft_metrics.png}
\caption{SFT training metrics over 1{,}000 fine-tuning steps on a
mixed Chinese--English instruction dataset: training loss, learning
rate, and gradient norm. The loss drops rapidly in the first 200
steps and then enters a slower decay phase. The learning rate
follows a WSD schedule (2\% warmup, stable, decay). Gradient norms stabilize
after approximately 300 steps, indicating that the fine-tuning
process has reached a stable optimization regime.}
\label{fig:sft_metrics}
\end{figure}

Figure~\ref{fig:sft_metrics} shows the supervised fine-tuning metrics
for the 1K-step SFT checkpoint used in the IFD analysis
(Appendix~\ref{app:ifd}). The training loss on the mixed
Chinese--English instruction dataset decreases from $\sim$2.5 to
$\sim$1.6 over 1{,}000 steps, with the gradient norm converging to a
stable range after the warmup phase.

\subsection{Direct Preference Optimization}
\label{sec:dpo}

Following supervised fine-tuning, we align the model with pairwise
human preferences via Direct Preference Optimization
(DPO)~\cite{rafailov2023dpo}.  DPO avoids explicit reward-model training
by optimizing the policy $\pi$ directly against a frozen reference
policy $\pi_{\text{ref}}$ (the SFT checkpoint).  For a prompt $x$ with
preferred response $y_w$ and dispreferred response $y_l$, the loss is:
\begin{equation}
\mathcal{L}_{\text{DPO}} = -\log \sigma\!\left(
\beta \Bigl[
\log\tfrac{\pi(y_w\mid x)}{\pi_{\text{ref}}(y_w\mid x)}
-\log\tfrac{\pi(y_l\mid x)}{\pi_{\text{ref}}(y_l\mid x)}
\Bigr]\right),
\end{equation}
where $\beta$ controls the KL-divergence penalty against the reference.
We use $\beta=0.1$, a batch of 64 preference pairs per step, and the
same hybrid Muon/AdamW optimizer.  Unlike the pretraining and SFT
phases, which both employ WSD (warmup--stable--decay) scheduling to
maintain a long high-learning-rate plateau, DPO alignment uses a
cosine schedule with short linear warmup.  The shorter ~1{,}500-step
alignment run does not benefit from an extended stable phase; instead,
cosine decay lowers the learning rate steadily, which discourages
over-optimisation away from the reference distribution and matches the
observed convergence pattern in Figure~\ref{fig:dpo_metrics}.

\begin{figure}[H]
\centering
\includegraphics[width=0.95\linewidth]{data/dpo_metrics.png}
\caption{DPO training metrics over $\sim$1{,}500 alignment steps:
preference loss (raw and 100-step moving average), learning rate
schedule, and gradient norm.}
\label{fig:dpo_metrics}
\end{figure}

Figure~\ref{fig:dpo_metrics} summarises the DPO training dynamics.
The raw preference loss (left panel) starts near $0.69$ and is visibly
noisy, a hallmark of pairwise preference sampling.  The 100-step moving
average reveals a steady downward trend that reaches a minimum of
$\sim$0.51 near step 1{,}200, after which it gently rebounds to
$\sim$0.52 and oscillates, suggesting the policy has converged to a
stable preference boundary rather than over-optimising away from the
reference distribution.  The learning-rate schedule (centre panel) peaks
at $5\times10^{-6}$ after a short linear warmup and then follows cosine
decay to a floor of $\sim$0.5\,$\times\,$10$^{-6}$.  Gradient norms (right panel) start
near 50 with occasional spikes above 55, then gradually decline and
stabilise in the 35--40 range after step 400, indicating consistent
gradient magnitudes throughout alignment.

% ======================================================================
\section{Numerical Stability via Residual Scaling}
\label{sec:num-stability}
% ======================================================================

Deep Transformers trained in BF16 face numerical stability challenges from
residual variance accumulation across layers. We evaluate the GPT-2
residual-scaling initialization~\cite{radford2019gpt2} as a mitigation
strategy.

\subsection{Variance Analysis}

At initialization with $\mathcal{N}(0, 0.02)$, a linear projection output has:
\begin{equation}
\Var(\mathbf{W}\mathbf{x}) = d_{\text{in}} \cdot (0.02)^2 \cdot \Var(\mathbf{x})
= 0.6144 \cdot \Var(\mathbf{x}) \quad (\text{for } d=1536).
\end{equation}

Within one block, attention and FFN each add a residual term. The variances
at each sub-stage are:

\begin{center}
\begin{tabular}{@{}lcc@{}}
\toprule
\textbf{Component} & \textbf{Operation} & $\Var$ (scaled by $\Var(\mathbf{x})$) \\
\midrule
Q/K/V proj & Linear(1536, $n_{\text{heads}}\cdot64$) & 0.6144 \\
Attention out & SDPA + $\mathbf{W}_o$ (scaled) & $0.378 / (2L)$ \\
Gate/Up proj & Linear(1536, 6912) & 0.6144 \\
SiLU gate & $\operatorname{SiLU}(z) \approx 0.5z$ & $0.6144 \times 0.298 = 0.1831$ \\
Gated product & element-wise $\odot$ & $\approx 0.6144 \times 0.1831 = 0.1125$ \\
Down proj & Linear(6912, 1536) (scaled) & $0.311 / (2L)$ \\
\midrule
Per-block residual & $\mathbf{R}_\ell = R_{\text{attn}} + R_{\text{ffn}}$ & $0.689 / (2L)$ (scaled) \\
\bottomrule
\end{tabular}
\end{center}

Without the $1/\sqrt{2L}$ factor on $\mathbf{W}_o$ and
$\mathbf{W}_{\text{down}}$, the per-block residual variance becomes $0.689$
instead of $0.689/(2L) \approx 0.014$. After $L=24$ blocks:
\begin{equation}
\begin{aligned}
\text{Without scaling: } \Var(\mathbf{x}_{24}) &\approx 1 + 24 \times 0.689 = 17.5,\\
\text{With scaling: } \Var(\mathbf{x}_{24}) &\approx 1 + 24 \times 0.014 = 1.34.
\end{aligned}
\end{equation}

\subsection{GPT-2 Residual Scaling}

The GPT-2 initialization~\cite{radford2019gpt2} scales output projections by
$1/\sqrt{2L}$:
\begin{equation}
\sigma_o = \sigma_{\text{down}} = 0.02 / \sqrt{2L}.
\end{equation}

This reduces per-block residual variance contribution from $0.689$ to
$0.689/L \approx 0.014$, a factor of $2L = 48$. The post-24-block variance
drops from $17.5$ to $1.34$, a $13.1\times$ improvement. In BF16
($7$-bit mantissa, ULP $= 0.0078$ at $w = 1.0$)~\cite{ieee754},
this keeps weight magnitudes within stable precision bounds. We further
recommend storing AdamW moments in FP32 and logging per-layer gradient
histograms during early training.

\subsection{Empirical Training Results}

Figure~\ref{fig:loss} shows both loss curves; GPT-2 residual scaling (lower
curve) maintains a clear advantage, particularly in the 0.3--0.8B token region.

\begin{table}[H]
\centering
\caption{Loss at 0.125B-interval milestones, 0--1B tokens.}
\label{tab:loss_milestones}
\begin{tabular}{@{}lccc@{}}
\toprule
\textbf{Tokens (B)} &
  \textbf{GPT-2 scaling} &
  \textbf{Kaiming init} &
  \textbf{$\Delta$} \\
\midrule
0.125 & 7.37 & 7.66 & 0.29 \\
0.250 & 5.80 & 6.14 & 0.34 \\
0.375 & 4.82 & 5.38 & 0.56 \\
0.500 & 4.06 & 4.80 & 0.74 \\
0.625 & 3.50 & 4.29 & 0.79 \\
0.750 & 3.24 & 3.80 & 0.56 \\
0.875 & 3.21 & 3.43 & 0.22 \\
1.000 & 2.80 & 3.18 & 0.38 \\
\bottomrule
\end{tabular}
\end{table}

Table~\ref{tab:loss_milestones} quantifies the per-milestone gap. GPT-2
residual scaling leads at every interval, with $\Delta$ growing from 0.29
at 0.125B to a peak of 0.79 at 0.625B, then narrowing to 0.38 at 1B.
The widening mid-range gap aligns with the variance accumulation region
identified in the theoretical analysis (Section~\ref{sec:num-stability}).

\subsection{Post-Training Weight Distribution}

To verify that the residual-scaling constraint persists throughout
training---not just at initialization---we compare weight value
distributions across three checkpoints:
\texttt{kami-15bt} (AdamW, GPT-2 residual scaling, 15B tokens), \texttt{norm-15bt} (AdamW, Normal
init, 15B tokens), and \texttt{muon-25bt} (Muon, 25B tokens).

Figure~\ref{fig:ckpt_weight_density} shows the kernel density
estimates. Training preserves the bimodal structure: residual-scaled
projections ($\mathbf{W}_o$, $\mathbf{W}_{\text{down}}$) remain
narrowly concentrated near zero, while non-scaled weights broaden
from their initial $\mathcal{N}(0,0.02)$ distribution. Two trends
are visible in the per-component weight std
(Table~\ref{tab:weight_std}, Appendix~\ref{app:weight_std}):

\begin{itemize}[nosep]
  \item \textbf{Training duration drives variance growth}: the
        \texttt{muon-25bt} checkpoint exhibits the largest weight std
        ($\sim$0.024 for attention projections), exceeding both 15B
        checkpoints ($\sim$0.015--0.021), reflecting continued weight
        drift from $\sigma_0 = 0.02$ as training progresses.
  \item \textbf{Residual scaling constrains early-stage drift}: at
        equal training budget (15B tokens) and with the same AdamW
        optimizer, the GPT-2-scaled \texttt{kami-15bt} shows
        \emph{smaller} std ($\sim$0.015) than the Normal-init
        \texttt{norm-15bt} ($\sim$0.021), indicating that residual
        scaling itself limits weight drift beyond its initialization
        effect. The Muon-trained \texttt{muon-25bt} (25B tokens)
        exhibits the largest std because cumulative training steps
        eventually overtake both initialization and per-step optimizer
        constraints.
\end{itemize}

Critically, the residual-scaled projections remain bounded at
$\sigma \approx 0.003$ across all checkpoints regardless of training
duration, confirming that the $1/\sqrt{2L}$ scaling continues to
enforce its design constraint throughout training.

\begin{figure}[H]
\centering
\includegraphics[width=0.95\linewidth]{data/ckpt_weight_density.png}
\caption{Weight value density estimates across three checkpoints,
grouped by component. Residual-scaled projections
($\mathbf{W}_o$, $\mathbf{W}_{\text{down}}$) maintain narrow
distributions near zero, while non-scaled weights show broader
spread.}
\label{fig:ckpt_weight_density}
\end{figure}

\begin{figure}[H]
\centering
\includegraphics[width=0.95\linewidth]{data/ckpt_weight_density_per_run.png}
\caption{Per-checkpoint weight density breakdowns.}
\label{fig:ckpt_weight_density_per_run}
\end{figure}

% ======================================================================
\section{Conclusion}
% ======================================================================

We have described the end-to-end pipeline for training a 1.2B Transformer with
{\sc AstrAI}: data preprocessing with JSON-driven tokenization and packing,
a 24-layer GQA-SwiGLU architecture, callback-based training with a hybrid
Muon/AdamW optimizer under DDP/FSDP executors, and cosine scheduling. We
further analyzed numerical stability under BF16, showing that GPT-2 residual
scaling ($\sigma_o = 0.02/\sqrt{2L}$) reduces per-block residual variance
by a factor of 48, keeping post-24-layer variance at $1.34$ versus $17.5$
without scaling. Post-training weight distribution analysis confirms that
this scaling constraint persists throughout training, with residual-scaled
projections maintaining narrow distributions ($\sigma \approx 0.003$)
regardless of optimizer or training duration.

Optimizer ablations (Figure~\ref{fig:ckpt_comparison}) demonstrate that
the hybrid Muon/AdamW configuration consistently outperforms pure AdamW
with both Kaiming and Normal initializations, achieving lower training loss
and faster convergence. Within the Muon family, applying AdamW to the
embedding layer rather than Muon yields a further loss reduction and more
stable gradient norms, confirming that 1D parameters benefit from
second-moment adaptation while 2D weight matrices gain from Muon's
orthogonalisation step. An SVD-based effective rank analysis
(Appendix~\ref{app:eff_rank}) further reveals that the model
operates near its representational capacity. The complete framework and
model weights are available at \url{https://github.com/ViperEkura/AstrAI}.

% ======================================================================
\appendix
% ======================================================================

% ======================================================================
\section{IFD Data Analysis}
\label{app:ifd}
% ======================================================================

Instruction Fulfillment Difficulty (IFD)~\cite{li2023ifd} compares
conditional and unconditional per-token losses:
\begin{equation}
\begin{aligned}
\mathrm{IFD} &= \frac{L_{\text{cond}}}{L_{\text{uncond}}},\\[2mm]
L_{\text{cond}} &= -\frac{1}{T}\sum_{t=1}^T \log P(y_t \mid \mathbf{x}, y_{<t}),\\[2mm]
L_{\text{uncond}} &= -\frac{1}{T}\sum_{t=1}^T \log P(y_t \mid y_{<t}).
\end{aligned}
\end{equation}
We compute IFD for $N=3000$ SFT samples (Alpaca-GPT4~\cite{alpaca})
using the base model (15B tokens) and the 1K-step SFT checkpoint.
After 1K SFT steps, both losses increase slightly; the mean IFD
changes from $0.8263$ (base) to $0.8485$ (1K SFT).

\subsection{Quantitative Summary}

Over $N=3000$ SFT samples from Alpaca-GPT4:
\begin{itemize}[nosep]
  \item \textbf{Base model}: mean IFD $= 0.8263$,
        median $= 0.8858$, std $= 0.1699$; $1.9\%$ of samples
        have IFD $> 1.0$.
  \item \textbf{1K SFT}: mean IFD $= 0.8485$,
        median $= 0.9083$, std $= 0.1588$; $3.1\%$ of samples
        exceed $1.0$.
  \item \textbf{Stability}: Pearson $r > 0.97$ between base and
        1K SFT IFD. The slight upward shift ($0.8263 \to 0.8485$)
        reflects both losses increasing after SFT, consistent with
        distribution shift during fine-tuning rather than uniform
        instruction-following improvement.
\end{itemize}

\subsection{Representative Samples}

Table~\ref{tab:ifd_examples} lists samples spanning the IFD range.

\begin{table}[H]
\centering
\caption{Representative IFD samples.}
\label{tab:ifd_examples}
\small
\begin{tabular}{@{}c c c c c p{4.2cm}@{}}
\toprule
\textbf{Idx} &
  \textbf{$L_{\text{cond}}^{\text{base}}$} &
  \textbf{$L_{\text{uncond}}^{\text{base}}$} &
  \textbf{$L_{\text{cond}}^{\text{1K}}$} &
  \textbf{$L_{\text{uncond}}^{\text{1K}}$} &
  \textbf{Instruction} \\
\midrule
81   & 13.38 & 5.84  & 13.25 & 5.69  & Classify incident as breach of protocol \\
906  & 13.12 & 9.75  & 13.06 & 9.75  & Convert numbers from words to digits \\
1076 & 2.53  & 2.46  & 2.53  & 2.53  & Pick best synonym \\
7    & 2.62  & 2.70  & 2.68  & 2.77  & Write a short story in third person \\
2427 & 2.59  & 2.84  & 2.69  & 2.90  & Find five most similar sentences \\
798  & 2.02  & 2.75  & 2.11  & 2.31  & List four social media platforms \\
223  & 1.34  & 3.16  & 1.36  & 3.27  & Classify text as Fiction or Non-fiction \\
\bottomrule
\end{tabular}
\end{table}

Samples with the highest conditional loss (rows~81,~906) are
short-answer classification tasks ($L_{\text{cond}} \approx 13$).
Lowest-IFD samples (row~223) are tasks where the instruction constrains
the output space so tightly that unconditional loss far exceeds
conditional loss. The four loss values remain nearly unchanged after
SFT across all samples.

\subsection{IFD Bias from Response Length}
\label{sec:ifd_bias}

Both losses are per-token averages. The variance of
$L_{\text{uncond}} = \frac{1}{T} \sum_{t=1}^T \log P(x_t)$
scales as $1/T$, so shorter responses produce noisier estimates.
Figure~\ref{fig:length_bias} plots the three metrics against response
length for the base model; samples with $<20$ tokens ($21.9\%$ of
the dataset) exhibit substantially higher scatter.

\begin{figure}[H]
\centering
\includegraphics[width=0.95\linewidth]{data/ifd_length_grid.png}
\caption{Response length vs.\ $L_{\text{cond}}$, $L_{\text{uncond}}$,
and IFD (base model, log scale on $x$-axis).}
\label{fig:length_bias}
\end{figure}

Table~\ref{tab:corr_bias} reports the correlations. Response length
is the dominant confound: $L_{\text{uncond}}$ shows a strong negative
monotonic trend ($\rho = -0.79$), while $L_{\text{cond}}$ is less
affected ($\rho = -0.48$). The net effect on IFD is a positive
correlation ($\rho = +0.72$).

\begin{table}[H]
\centering
\caption{Pearson $r$ and Spearman $\rho$ between sample dimensions and IFD components (base model).}
\label{tab:corr_bias}
\small
\begin{tabular}{@{}lcccccc@{}}
\toprule
 & \multicolumn{2}{c}{vs.\ $L_{\text{cond}}$}
 & \multicolumn{2}{c}{vs.\ $L_{\text{uncond}}$}
 & \multicolumn{2}{c}{vs.\ IFD} \\
\cmidrule(lr){2-3} \cmidrule(lr){4-5} \cmidrule(lr){6-7}
\textbf{Dimension} & $r$ & $\rho$ & $r$ & $\rho$ & $r$ & $\rho$ \\
\midrule
Instruction length & $+0.07$ & $+0.06$ & $+0.15$ & $+0.24$ & $-0.25$ & $-0.34$ \\
Response length    & $-0.36$ & $-0.48$ & $-0.56$ & $-0.79$ & $+0.58$ & $+0.72$ \\
\bottomrule
\end{tabular}
\end{table}

\subsection{Loss Ratio}
\label{sec:loss_ratio}

We further define the \textbf{Loss Ratio} as the fraction of conditional
loss retained after SFT:
\begin{equation}
\text{Loss Ratio} = \frac{L_{\text{cond}}^{\text{1K}}}{L_{\text{cond}}^{\text{base}}}.
\end{equation}

Over $N=3000$ samples:
\begin{itemize}[nosep]
  \item Mean $= 1.106$, median $= 1.084$, std $= 0.110$;
        $90.8\%$ of samples exceed $1.0$.
  \item Range: $[0.768, 1.962]$; only $9.2\%$ of samples show
        a decrease ($<1.0$) in conditional loss after SFT.
\end{itemize}

The predominance of loss ratio $>1$ confirms that the 1K-step SFT
checkpoint has not converged to a lower-loss region for the
evaluation samples. Instead, the fine-tuning distribution shift
increases NLL on most held-out instructions.

Table~\ref{tab:ifd_lr_corr} reports the pairwise correlations.
Although IFD\textsubscript{ckpt} and Loss Ratio both depend on
$L_{\text{cond}}^{\text{1K}}$, they need not correlate because their
denominators vary independently across samples. The observed correlation
is near zero ($r = -0.02$, $\rho = 0.04$), precisely because the
{\em relative} ordering of $L_{\text{cond}}^{\text{base}}$ and
$L_{\text{uncond}}^{\text{ckpt}}$ (which determine the slope
$k_i = L_{\text{cond},i}^{\text{base}} / L_{\text{uncond},i}^{\text{ckpt}}$
in the relationship $\text{IFD}_{\text{ckpt},i} = k_i \cdot
\text{Loss Ratio}_i$) varies widely, breaking the proportionality
at the sample level. This invalidates the naive expectation that a shared
numerator guarantees correlation~\cite{li2023ifd}.

\begin{table}[H]
\centering
\caption{Pairwise correlations between IFD variants and Loss Ratio.}
\label{tab:ifd_lr_corr}
\small
\begin{tabular}{@{}lcc@{}}
\toprule
\textbf{Pair} & Pearson $r$ & Spearman $\rho$ \\
\midrule
IFD\textsubscript{base} vs.\ IFD\textsubscript{ckpt}  & $+0.97$ & $+0.96$ \\
IFD\textsubscript{base} vs.\ Loss Ratio       & $-0.15$ & $-0.06$ \\
IFD\textsubscript{ckpt} vs.\ Loss Ratio       & $-0.02$ & $+0.04$ \\
\bottomrule
\end{tabular}
\end{table}

The near-perfect correlation between IFD\textsubscript{base} and
IFD\textsubscript{ckpt} ($r = 0.97$) reveals that the IFD ranking is
highly robust to the choice of evaluation model: samples that the
base model finds difficult remain difficult after 1K SFT steps.
This stability justifies using the base-model IFD as a data
selection signal without re-evaluating after fine-tuning.

% ======================================================================
\section{Effective Rank Analysis}
\label{app:eff_rank}
% ======================================================================

To assess how well the trained parameters utilize their allocated
capacity, we perform an SVD-based effective rank analysis on three
checkpoints: \texttt{kami-15bt} (Muon, 15B tokens), \texttt{norm-15bt}
(Normal init, 15B tokens), and \texttt{muon-25bt} (Muon, 25B tokens).
For each 2D weight matrix $\mathbf{W} \in \mathbb{R}^{m\times n}$ with
SVD $\mathbf{W} = \mathbf{U}\boldsymbol{\Sigma}\mathbf{V}^{\mkern-1mu\mathsf{T}}$,
we compute the effective rank at 99\% energy:
\begin{equation}
\text{ER@99\%} = \frac{1}{\min(m,n)} \min_k \left\{ k \;\middle|\; \frac{\sum_{i=1}^{k} \sigma_i^2}{\sum_{i=1}^{\min(m,n)} \sigma_i^2} \ge 0.99 \right\}.
\end{equation}

Table~\ref{tab:eff_rank} summarizes the results. All three checkpoints
exhibit a high overall ER@99\% ($\sim$90\%), indicating that the 1.2B
model operates close to its representational capacity. Key findings:

\begin{itemize}[nosep]
  \item Q/O projections show lower ER@99\% ($\sim$0.73--0.77) and
        high condition numbers ($\kappa > 10^4$), consistent with the
        low-rank structure of GQA (24 query heads sharing 4 KV heads).
  \item K/V projections, FFN layers, and embeddings maintain high
        ER@99\% ($\sim$0.96--0.98) and low condition numbers
        ($\kappa < 10$).
  \item The overall ER@99\% varies by less than 0.01 across
        checkpoints, indicating these properties are determined
        primarily by architecture rather than optimizer or training
        duration.
\end{itemize}

\begin{table}[H]
\centering
\caption{SVD effective rank (ER@99\%) and mean condition number
($\kappa$) by component across three checkpoints.}
\label{tab:eff_rank}
\small
\begin{tabular}{@{}lcccccc@{}}
\toprule
& \multicolumn{3}{c}{\textbf{ER@99\%}}
& \multicolumn{3}{c}{\textbf{Cond.\ Number $\kappa$}} \\
\cmidrule(lr){2-4} \cmidrule(lr){5-7}
\textbf{Component} & \textbf{kami} & \textbf{norm} & \textbf{muon}
& \textbf{kami} & \textbf{norm} & \textbf{muon} \\
\midrule
attn.k\_proj         & 0.967 & 0.971 & 0.960 & 8.4     & 5.4     & 5.7 \\
attn.o\_proj         & 0.766 & 0.708 & 0.730 & 31{,}589 & 57{,}284 & 22{,}838 \\
attn.q\_proj          & 0.754 & 0.764 & 0.756 & 46{,}644 & 32{,}072 & 14{,}472 \\
attn.v\_proj          & 0.976 & 0.976 & 0.971 & 2.5     & 2.4     & 2.9 \\
embed\_tokens         & 0.985 & 0.987 & 0.984 & 4.9     & 1.9     & 3.5 \\
lm\_head              & 0.969 & 0.981 & 0.980 & 21.0    & 13.5    & 16.1 \\
mlp.down              & 0.961 & 0.961 & 0.963 & 6.4     & 7.2     & 7.2 \\
mlp.gate              & 0.966 & 0.967 & 0.965 & 5.7     & 5.3     & 5.7 \\
mlp.up                & 0.967 & 0.968 & 0.965 & 4.9     & 5.9     & 4.9 \\
\midrule
\textbf{Overall ER@99\%} & \textbf{0.909} & \textbf{0.903} & \textbf{0.903} & & & \\
\bottomrule
\end{tabular}
\end{table}

% ======================================================================
\section{Per-Component Weight Statistics}
\label{app:weight_std}
% ======================================================================

Table~\ref{tab:weight_std} reports the weight standard deviation by
component for each checkpoint, supplementing the post-training weight
distribution analysis in Section~\ref{sec:num-stability}.

\begin{table}[H]
\centering
\caption{Weight std by component across checkpoints. Non-scaled
weights broaden with training duration; Muon constrains drift at
equal token count (15B) but is overtaken by longer training (25B).
Residual-scaled projections ($\mathbf{W}_o$, $\mathbf{W}_{\text{down}}$)
remain bounded.}
\label{tab:weight_std}
\small
\begin{tabular}{@{}lccc@{}}
\toprule
\textbf{Component} & \textbf{kami-15bt} & \textbf{norm-15bt} & \textbf{muon-25bt} \\
 & (AdamW, 15B) & (AdamW, 15B) & (Muon, 25B) \\
\midrule
attn.q\_proj          & 0.0154 & 0.0207 & 0.0230 \\
attn.k\_proj          & 0.0153 & 0.0206 & 0.0238 \\
attn.v\_proj          & 0.0146 & 0.0202 & 0.0244 \\
attn.o\_proj$^*$      & 0.0148 & 0.0084 & 0.0177 \\
mlp.up                & 0.0153 & 0.0204 & 0.0237 \\
mlp.gate              & 0.0155 & 0.0204 & 0.0235 \\
mlp.down$^*$          & 0.0100 & 0.0089 & 0.0180 \\
embed\_tokens         & 0.0205 & 0.0205 & 0.0239 \\
lm\_head              & 0.0224 & 0.0257 & 0.0298 \\
\bottomrule
\end{tabular}
\\[2pt]
\footnotesize $^*$Residual-scaled projection ($\sigma_0 = 0.02/\sqrt{2L}$).
\end{table}

% ======================================================================
\begin{thebibliography}{99}

\bibitem{alpaca}
R.~Taori, I.~Gulrajani, T.~Zhang, Y.~Dubois, X.~Li, C.~Guestrin,
P.~Liang, T.~B.~Hashimoto.
Alpaca: A strong, replicable instruction-following model.
\textit{Stanford Center for Research on Foundation Models (CRFM)}, 2023.

\bibitem{ainslie2023gqa}
J.~Ainslie, J.~Lee-Thorp, M.~de Jong, Y.~Zemlyanskiy, F.~Lebr\'on, S.~Sanghai.
GQA: Training generalized multi-query transformer models from multi-head
checkpoints. \textit{EMNLP}, 2023.

\bibitem{alembic}
Alembic Contributors. \textit{Alembic: A lightweight LLM-driven SFT data
generation, cleaning, and scoring pipeline.}
\url{https://github.com/ViperEkura/Alembic}, 2026.

\bibitem{astrai}
AstrAI Contributors. \textit{AstrAI: An open-source training and inference
framework for Transformer language models.}
\url{https://github.com/ViperEkura/AstrAI}, 2026.

\bibitem{broder1997syntactic}
A.~Z.~Broder. On the resemblance and containment of documents.
\textit{SEQUENCES '97}, 1997.

\bibitem{li2023ifd}
M.~Li, Y.~Zhang, Z.~Li, J.~Chen, L.~Chen, N.~Cheng, J.~Wang, T.~Zhou, J.~Xiao.
From quantity to quality: Boosting LLM performance with self-guided data
selection for instruction tuning.
\textit{NAACL}, 2024.

\bibitem{ieee754}
IEEE Computer Society. \textit{IEEE Standard for Floating-Point Arithmetic},
IEEE Std 754-2019, 2019.

\bibitem{loshchilov2019adamw}
I.~Loshchilov, F.~Hutter. Decoupled weight decay regularization.
\textit{ICLR}, 2019.

\bibitem{radford2019gpt2}
A.~Radford, J.~Wu, R.~Child, D.~Luan, D.~Amodei, I.~Sutskever.
Language models are unsupervised multitask learners.
\textit{OpenAI Blog}, 2019.

\bibitem{rafailov2023dpo}
R.~Rafailov, A.~Sharma, E.~Mitchell, C.~D.~Manning, S.~Ermon, C.~Finn.
Direct Preference Optimization: Your language model is secretly a reward model.
\textit{NeurIPS}, 2023.

\bibitem{shazeer2020glu}
N.~Shazeer. GLU variants improve Transformer.
\textit{arXiv:2002.05202}, 2020.

\bibitem{su2024roformer}
J.~Su, A.~Murtadha, Y.~Lu, S.~Pan, B.~Wen, Y.~Liu.
Roformer: Enhanced transformer with rotary position embedding.
\textit{Neurocomputing}, 568:127063, 2024.

\end{thebibliography}

\end{document}