refactor : metric_logger 改用事件类型 (type=step/validation/epoch)

- 每种事件独立 schema,不再混入 null 字段
- 回调顺序 validation 移到 metric_logger 之前,确保 on_optimizer_step 先跑
- 用内部 _last_val_loss 代替 TrainContext.last_val_iter 判断新验证
- 修复 factory.py 未使用导入、evaluate_ifeval.py 多余 f 前缀
This commit is contained in:
ViperEkura 2026-06-25 17:18:20 +08:00
parent 88ec63121d
commit b4587c5d08
4 changed files with 30 additions and 22 deletions

View File

@ -4,7 +4,6 @@ import inspect
import sys import sys
from abc import ABC from abc import ABC
from typing import ( from typing import (
Any,
Callable, Callable,
Dict, Dict,
ForwardRef, ForwardRef,

View File

@ -237,6 +237,7 @@ class MetricLoggerCallback(TrainCallback):
metrics: List[str] = None, metrics: List[str] = None,
): ):
self.last_log_iter = 0 self.last_log_iter = 0
self._last_val_loss = None
self.save_interval = save_interval self.save_interval = save_interval
self.log_interval = log_interval self.log_interval = log_interval
self.metrics = metrics or ["loss", "lr"] self.metrics = metrics or ["loss", "lr"]
@ -258,46 +259,54 @@ class MetricLoggerCallback(TrainCallback):
"grad_nan_num": ctx_get_grad_nan_num, "grad_nan_num": ctx_get_grad_nan_num,
} }
def _get_log_data(self, context: TrainContext): def _metrics(self, context: TrainContext, names):
data = { return {
m: self._metric_funcs[m](context)
for m in names
if self._metric_funcs[m](context) is not None
}
@only_on_rank(0)
def _append(self, event_type: str, context: TrainContext, **extra):
entry = {
"type": event_type,
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
"epoch": context.epoch, "epoch": context.epoch,
"iter": context.iteration, "iter": context.iteration,
**extra,
} }
for m in self.metrics: self.log_cache.append(entry)
val = self._metric_funcs[m](context)
if val is not None:
data[m] = val
return data
@only_on_rank(0) @only_on_rank(0)
def _add_log(self, log_data): def _flush(self, epoch, iter):
self.log_cache.append(log_data)
@only_on_rank(0)
def _save_log(self, epoch, iter):
log_file = self.log_dir / f"epoch_{epoch}_iter_{iter}_metric.jsonl" log_file = self.log_dir / f"epoch_{epoch}_iter_{iter}_metric.jsonl"
log_file.parent.mkdir(parents=True, exist_ok=True) log_file.parent.mkdir(parents=True, exist_ok=True)
with open(log_file, "w") as f: with open(log_file, "w") as f:
for log in self.log_cache: for log in self.log_cache:
f.write(json.dumps(log) + "\n") f.write(json.dumps(log) + "\n")
def on_batch_end(self, context): def on_batch_end(self, context):
if context.iteration % self.log_interval == 0: if context.iteration % self.log_interval == 0:
log_data = self._get_log_data(context) step_metrics = [m for m in self.metrics if m != "val_loss"]
self._add_log(log_data) self._append("step", context, **self._metrics(context, step_metrics))
if context.iteration - self.last_log_iter >= self.save_interval: if context.iteration - self.last_log_iter >= self.save_interval:
self._save_log(context.epoch, context.iteration) self._flush(context.epoch, context.iteration)
self.last_log_iter = context.iteration self.last_log_iter = context.iteration
def on_optimizer_step(self, context):
if context.val_loss is not None and context.val_loss != self._last_val_loss:
self._append("validation", context, val_loss=context.val_loss)
self._last_val_loss = context.val_loss
def on_epoch_end(self, context):
self._append("epoch", context)
def on_train_end(self, context): def on_train_end(self, context):
if context.iteration != self.last_log_iter: if context.iteration != self.last_log_iter:
self._save_log(context.epoch, context.iteration) self._flush(context.epoch, context.iteration)
def on_error(self, context): def on_error(self, context):
self._save_log(context.epoch, context.iteration) self._flush(context.epoch, context.iteration)
@CallbackFactory.register("validation") @CallbackFactory.register("validation")

View File

@ -34,6 +34,7 @@ class Trainer:
cfg.ckpt_dir, cfg.ckpt_dir,
cfg.ckpt_interval, cfg.ckpt_interval,
), ),
CallbackFactory.create("validation"),
CallbackFactory.create( CallbackFactory.create(
"metric_logger", "metric_logger",
log_dir=cfg.log_dir, log_dir=cfg.log_dir,
@ -43,7 +44,6 @@ class Trainer:
), ),
CallbackFactory.create("progress_bar", cfg.n_epoch), CallbackFactory.create("progress_bar", cfg.n_epoch),
CallbackFactory.create("gradient_clipping", cfg.max_grad_norm), CallbackFactory.create("gradient_clipping", cfg.max_grad_norm),
CallbackFactory.create("validation"),
] ]
return callbacks return callbacks

View File

@ -571,7 +571,7 @@ def main():
print(f" Unsupported: {summary['unsupported_constraints']}") print(f" Unsupported: {summary['unsupported_constraints']}")
print(f"{'=' * 60}") print(f"{'=' * 60}")
print(f"\nPer-type accuracy:") print("\nPer-type accuracy:")
for inst_id, stats in sorted(summary["per_type_accuracy"].items()): for inst_id, stats in sorted(summary["per_type_accuracy"].items()):
print( print(
f" {inst_id:50s} {stats['accuracy']:.2%} " f" {inst_id:50s} {stats['accuracy']:.2%} "