refactor : replace iteration with consumed_samples

- Replace context.iteration with consumed_samples (global sample count)
- Add optimizer_step property derived from consumed_samples
- Checkpoint meta.json stores consumed_samples, drops iteration
- CLI --start_batch renamed to --start_samples (per-rank samples)
- Checkpoint dir naming: epoch_X_step_Y instead of epoch_X_iter_Y
- Metric log entries use step and consumed_samples fields
- Backward compat removed (old iteration checkpoints unsupported)
This commit is contained in:
2026-06-30 18:42:42 +08:00
parent 44579ea6dc
commit aabb0d83e9
10 changed files with 72 additions and 48 deletions
+1 -1
View File
@@ -75,7 +75,7 @@ class MultiTurnDataset(Dataset):
class EarlyStoppingDataset(Dataset):
"""Dataset that triggers early stopping after a specified number of iterations."""
"""Dataset that triggers early stopping after consuming a specified number of samples."""
def __init__(self, length=10, stop_after=5):
self.length = length
+9 -4
View File
@@ -25,7 +25,9 @@ def test_single_process():
scheduler.step()
checkpoint = Checkpoint(state_dict=model.state_dict(), epoch=3, iteration=30)
checkpoint = Checkpoint(
state_dict=model.state_dict(), epoch=3, consumed_samples=120
)
with tempfile.TemporaryDirectory() as tmpdir:
checkpoint.save(tmpdir)
@@ -33,7 +35,7 @@ def test_single_process():
loaded_checkpoint = Checkpoint.load(tmpdir)
assert loaded_checkpoint.epoch == 3
assert loaded_checkpoint.iteration == 30
assert loaded_checkpoint.consumed_samples == 120
def test_checkpoint_with_extra():
@@ -46,7 +48,10 @@ def test_checkpoint_with_extra():
"scheduler": {"last_epoch": 5},
}
checkpoint = Checkpoint(
state_dict=model.state_dict(), epoch=1, iteration=10, extra=extra
state_dict=model.state_dict(),
epoch=1,
consumed_samples=40,
extra=extra,
)
with tempfile.TemporaryDirectory() as tmpdir:
@@ -77,7 +82,7 @@ def simple_training():
checkpoint = Checkpoint(
state_dict=model.state_dict(),
epoch=2,
iteration=10,
consumed_samples=40,
)
rank = get_rank()
+1 -1
View File
@@ -52,7 +52,7 @@ def create_train_config(
batch_per_device: Batch size per device (default: 2)
grad_accum_steps: Gradient accumulation steps (default: 1)
max_grad_norm: Maximum gradient norm for clipping (default: 1.0)
ckpt_interval: Checkpoint save interval in iterations (default: 5)
ckpt_interval: Checkpoint save interval in optimizer steps (default: 5)
random_seed: Random seed for reproducibility (default: 42)
**kwargs: Additional arguments passed to TrainConfig
+4 -4
View File
@@ -44,14 +44,14 @@ def test_early_stopping_simulation(base_test_env, early_stopping_dataset):
pass
# Resume from latest checkpoint
load_dir = os.path.join(base_test_env["test_dir"], "epoch_0_iter_2")
load_dir = os.path.join(base_test_env["test_dir"], "epoch_0_step_1")
trainer = Trainer(train_config)
trainer.train(resume_dir=load_dir)
# Verify checkpoint was saved at expected iteration
load_dir = os.path.join(base_test_env["test_dir"], "epoch_1_iter_10")
# Verify checkpoint was saved at expected step
load_dir = os.path.join(base_test_env["test_dir"], "epoch_1_step_5")
import json
with open(os.path.join(load_dir, "meta.json")) as f:
meta = json.load(f)
assert meta["iteration"] == 10
assert meta["consumed_samples"] == 20