feat: add grad_snr metric with EMA-based gradient SNR tracking
- add GradSNRTracker to metric_util.py computing SNR = E[g]^2 / Var(g) via per-parameter EMA moments - add grad_snr_tracker field to TrainContext (instantiated by default) - register grad_snr in MetricCallback, update tracker on each optimizer step before metrics are recorded - add grad_snr to default --metrics in train.py CLI
This commit is contained in:
@@ -17,6 +17,7 @@ from astrai.parallel.setup import get_current_device, get_rank, get_world_size
|
||||
from astrai.protocols import OptimizerProtocol, SchedulerProtocol
|
||||
from astrai.serialization import Checkpoint, load_json
|
||||
from astrai.tokenize import AutoTokenizer
|
||||
from astrai.trainer.metric_util import GradSNRTracker
|
||||
from astrai.trainer.rollout import RolloutGenerator, RolloutRunner
|
||||
from astrai.trainer.strategy import BaseStrategy, StrategyFactory
|
||||
|
||||
@@ -38,6 +39,7 @@ class TrainContext:
|
||||
consumed_samples: int = field(default=0)
|
||||
loss: float = field(default=0.0)
|
||||
grad_norm: Optional[float] = field(default=None)
|
||||
grad_snr_tracker: GradSNRTracker = field(default_factory=GradSNRTracker)
|
||||
val_dataloader: Optional[DataLoader] = field(default=None)
|
||||
val_loss: Optional[float] = field(default=None)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user