refactor : metric_logger 改用事件类型 (type=step/validation/epoch)

- 每种事件独立 schema,不再混入 null 字段
- 回调顺序 validation 移到 metric_logger 之前,确保 on_optimizer_step 先跑
- 用内部 _last_val_loss 代替 TrainContext.last_val_iter 判断新验证
- 修复 factory.py 未使用导入、evaluate_ifeval.py 多余 f 前缀
This commit is contained in:
ViperEkura 2026-06-25 17:18:20 +08:00
parent 88ec63121d
commit b4587c5d08
4 changed files with 30 additions and 22 deletions

View File

@ -4,7 +4,6 @@ import inspect
import sys
from abc import ABC
from typing import (
Any,
Callable,
Dict,
ForwardRef,

View File

@ -237,6 +237,7 @@ class MetricLoggerCallback(TrainCallback):
metrics: List[str] = None,
):
self.last_log_iter = 0
self._last_val_loss = None
self.save_interval = save_interval
self.log_interval = log_interval
self.metrics = metrics or ["loss", "lr"]
@ -258,46 +259,54 @@ class MetricLoggerCallback(TrainCallback):
"grad_nan_num": ctx_get_grad_nan_num,
}
def _get_log_data(self, context: TrainContext):
data = {
def _metrics(self, context: TrainContext, names):
return {
m: self._metric_funcs[m](context)
for m in names
if self._metric_funcs[m](context) is not None
}
@only_on_rank(0)
def _append(self, event_type: str, context: TrainContext, **extra):
entry = {
"type": event_type,
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
"epoch": context.epoch,
"iter": context.iteration,
**extra,
}
for m in self.metrics:
val = self._metric_funcs[m](context)
if val is not None:
data[m] = val
return data
self.log_cache.append(entry)
@only_on_rank(0)
def _add_log(self, log_data):
self.log_cache.append(log_data)
@only_on_rank(0)
def _save_log(self, epoch, iter):
def _flush(self, epoch, iter):
log_file = self.log_dir / f"epoch_{epoch}_iter_{iter}_metric.jsonl"
log_file.parent.mkdir(parents=True, exist_ok=True)
with open(log_file, "w") as f:
for log in self.log_cache:
f.write(json.dumps(log) + "\n")
def on_batch_end(self, context):
if context.iteration % self.log_interval == 0:
log_data = self._get_log_data(context)
self._add_log(log_data)
step_metrics = [m for m in self.metrics if m != "val_loss"]
self._append("step", context, **self._metrics(context, step_metrics))
if context.iteration - self.last_log_iter >= self.save_interval:
self._save_log(context.epoch, context.iteration)
self._flush(context.epoch, context.iteration)
self.last_log_iter = context.iteration
def on_optimizer_step(self, context):
if context.val_loss is not None and context.val_loss != self._last_val_loss:
self._append("validation", context, val_loss=context.val_loss)
self._last_val_loss = context.val_loss
def on_epoch_end(self, context):
self._append("epoch", context)
def on_train_end(self, context):
if context.iteration != self.last_log_iter:
self._save_log(context.epoch, context.iteration)
self._flush(context.epoch, context.iteration)
def on_error(self, context):
self._save_log(context.epoch, context.iteration)
self._flush(context.epoch, context.iteration)
@CallbackFactory.register("validation")

View File

@ -34,6 +34,7 @@ class Trainer:
cfg.ckpt_dir,
cfg.ckpt_interval,
),
CallbackFactory.create("validation"),
CallbackFactory.create(
"metric_logger",
log_dir=cfg.log_dir,
@ -43,7 +44,6 @@ class Trainer:
),
CallbackFactory.create("progress_bar", cfg.n_epoch),
CallbackFactory.create("gradient_clipping", cfg.max_grad_norm),
CallbackFactory.create("validation"),
]
return callbacks

View File

@ -571,7 +571,7 @@ def main():
print(f" Unsupported: {summary['unsupported_constraints']}")
print(f"{'=' * 60}")
print(f"\nPer-type accuracy:")
print("\nPer-type accuracy:")
for inst_id, stats in sorted(summary["per_type_accuracy"].items()):
print(
f" {inst_id:50s} {stats['accuracy']:.2%} "