refactor : metric_logger 改用事件类型 (type=step/validation/epoch)
- 每种事件独立 schema,不再混入 null 字段 - 回调顺序 validation 移到 metric_logger 之前,确保 on_optimizer_step 先跑 - 用内部 _last_val_loss 代替 TrainContext.last_val_iter 判断新验证 - 修复 factory.py 未使用导入、evaluate_ifeval.py 多余 f 前缀
This commit is contained in:
parent
88ec63121d
commit
b4587c5d08
|
|
@ -4,7 +4,6 @@ import inspect
|
||||||
import sys
|
import sys
|
||||||
from abc import ABC
|
from abc import ABC
|
||||||
from typing import (
|
from typing import (
|
||||||
Any,
|
|
||||||
Callable,
|
Callable,
|
||||||
Dict,
|
Dict,
|
||||||
ForwardRef,
|
ForwardRef,
|
||||||
|
|
|
||||||
|
|
@ -237,6 +237,7 @@ class MetricLoggerCallback(TrainCallback):
|
||||||
metrics: List[str] = None,
|
metrics: List[str] = None,
|
||||||
):
|
):
|
||||||
self.last_log_iter = 0
|
self.last_log_iter = 0
|
||||||
|
self._last_val_loss = None
|
||||||
self.save_interval = save_interval
|
self.save_interval = save_interval
|
||||||
self.log_interval = log_interval
|
self.log_interval = log_interval
|
||||||
self.metrics = metrics or ["loss", "lr"]
|
self.metrics = metrics or ["loss", "lr"]
|
||||||
|
|
@ -258,46 +259,54 @@ class MetricLoggerCallback(TrainCallback):
|
||||||
"grad_nan_num": ctx_get_grad_nan_num,
|
"grad_nan_num": ctx_get_grad_nan_num,
|
||||||
}
|
}
|
||||||
|
|
||||||
def _get_log_data(self, context: TrainContext):
|
def _metrics(self, context: TrainContext, names):
|
||||||
data = {
|
return {
|
||||||
|
m: self._metric_funcs[m](context)
|
||||||
|
for m in names
|
||||||
|
if self._metric_funcs[m](context) is not None
|
||||||
|
}
|
||||||
|
|
||||||
|
@only_on_rank(0)
|
||||||
|
def _append(self, event_type: str, context: TrainContext, **extra):
|
||||||
|
entry = {
|
||||||
|
"type": event_type,
|
||||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||||
"epoch": context.epoch,
|
"epoch": context.epoch,
|
||||||
"iter": context.iteration,
|
"iter": context.iteration,
|
||||||
|
**extra,
|
||||||
}
|
}
|
||||||
for m in self.metrics:
|
self.log_cache.append(entry)
|
||||||
val = self._metric_funcs[m](context)
|
|
||||||
if val is not None:
|
|
||||||
data[m] = val
|
|
||||||
return data
|
|
||||||
|
|
||||||
@only_on_rank(0)
|
@only_on_rank(0)
|
||||||
def _add_log(self, log_data):
|
def _flush(self, epoch, iter):
|
||||||
self.log_cache.append(log_data)
|
|
||||||
|
|
||||||
@only_on_rank(0)
|
|
||||||
def _save_log(self, epoch, iter):
|
|
||||||
log_file = self.log_dir / f"epoch_{epoch}_iter_{iter}_metric.jsonl"
|
log_file = self.log_dir / f"epoch_{epoch}_iter_{iter}_metric.jsonl"
|
||||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
with open(log_file, "w") as f:
|
with open(log_file, "w") as f:
|
||||||
for log in self.log_cache:
|
for log in self.log_cache:
|
||||||
f.write(json.dumps(log) + "\n")
|
f.write(json.dumps(log) + "\n")
|
||||||
|
|
||||||
def on_batch_end(self, context):
|
def on_batch_end(self, context):
|
||||||
if context.iteration % self.log_interval == 0:
|
if context.iteration % self.log_interval == 0:
|
||||||
log_data = self._get_log_data(context)
|
step_metrics = [m for m in self.metrics if m != "val_loss"]
|
||||||
self._add_log(log_data)
|
self._append("step", context, **self._metrics(context, step_metrics))
|
||||||
|
|
||||||
if context.iteration - self.last_log_iter >= self.save_interval:
|
if context.iteration - self.last_log_iter >= self.save_interval:
|
||||||
self._save_log(context.epoch, context.iteration)
|
self._flush(context.epoch, context.iteration)
|
||||||
self.last_log_iter = context.iteration
|
self.last_log_iter = context.iteration
|
||||||
|
|
||||||
|
def on_optimizer_step(self, context):
|
||||||
|
if context.val_loss is not None and context.val_loss != self._last_val_loss:
|
||||||
|
self._append("validation", context, val_loss=context.val_loss)
|
||||||
|
self._last_val_loss = context.val_loss
|
||||||
|
|
||||||
|
def on_epoch_end(self, context):
|
||||||
|
self._append("epoch", context)
|
||||||
|
|
||||||
def on_train_end(self, context):
|
def on_train_end(self, context):
|
||||||
if context.iteration != self.last_log_iter:
|
if context.iteration != self.last_log_iter:
|
||||||
self._save_log(context.epoch, context.iteration)
|
self._flush(context.epoch, context.iteration)
|
||||||
|
|
||||||
def on_error(self, context):
|
def on_error(self, context):
|
||||||
self._save_log(context.epoch, context.iteration)
|
self._flush(context.epoch, context.iteration)
|
||||||
|
|
||||||
|
|
||||||
@CallbackFactory.register("validation")
|
@CallbackFactory.register("validation")
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,7 @@ class Trainer:
|
||||||
cfg.ckpt_dir,
|
cfg.ckpt_dir,
|
||||||
cfg.ckpt_interval,
|
cfg.ckpt_interval,
|
||||||
),
|
),
|
||||||
|
CallbackFactory.create("validation"),
|
||||||
CallbackFactory.create(
|
CallbackFactory.create(
|
||||||
"metric_logger",
|
"metric_logger",
|
||||||
log_dir=cfg.log_dir,
|
log_dir=cfg.log_dir,
|
||||||
|
|
@ -43,7 +44,6 @@ class Trainer:
|
||||||
),
|
),
|
||||||
CallbackFactory.create("progress_bar", cfg.n_epoch),
|
CallbackFactory.create("progress_bar", cfg.n_epoch),
|
||||||
CallbackFactory.create("gradient_clipping", cfg.max_grad_norm),
|
CallbackFactory.create("gradient_clipping", cfg.max_grad_norm),
|
||||||
CallbackFactory.create("validation"),
|
|
||||||
]
|
]
|
||||||
return callbacks
|
return callbacks
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -571,7 +571,7 @@ def main():
|
||||||
print(f" Unsupported: {summary['unsupported_constraints']}")
|
print(f" Unsupported: {summary['unsupported_constraints']}")
|
||||||
print(f"{'=' * 60}")
|
print(f"{'=' * 60}")
|
||||||
|
|
||||||
print(f"\nPer-type accuracy:")
|
print("\nPer-type accuracy:")
|
||||||
for inst_id, stats in sorted(summary["per_type_accuracy"].items()):
|
for inst_id, stats in sorted(summary["per_type_accuracy"].items()):
|
||||||
print(
|
print(
|
||||||
f" {inst_id:50s} {stats['accuracy']:.2%} "
|
f" {inst_id:50s} {stats['accuracy']:.2%} "
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue