feat: version rollout weight updates

Track a monotonic policy version across optimizer steps, scheduler updates, and rollout results. Serialize synchronous generation with weight acknowledgements and invalidate reusable prefix KV entries so cached samples remain attributable to the behavior policy that generated them.

tests/trainer/test_online_strategy.py

@@ -43,6 +43,8 @@ class _RecordingRunner:
         self.calls = 0
         self.step_calls = 0
         self._fresh = True
+        self.policy_version = result.policy_version
+        self.weight_updates = []
 
     def __call__(self, batch):
         self.calls += 1
@@ -53,6 +55,11 @@ class _RecordingRunner:
     def step(self):
         self.step_calls += 1
 
+    def update_weights(self, policy_version):
+        self.policy_version = policy_version
+        self.weight_updates.append(policy_version)
+        return policy_version
+
     def swap_result(self, result):
         self.result = result
         self._fresh = True
@@ -302,6 +309,8 @@ def test_step_called_when_sync_gradients_true(device):
     strat({"input_ids": torch.randint(3, 200, (2, 4), device=device)})
     strat.on_optimizer_step()
     assert runner.step_calls == 1
+    assert runner.weight_updates == [1]
+    assert strat.policy_version == 1
 
 
 def test_loss_is_differentiable_dpo(device):

tests/trainer/test_rollout.py

@@ -65,6 +65,7 @@ def test_raw_rollout_fields():
     )
     assert r.prompts.shape == (2, 4)
     assert r.responses.shape == (2, 3, 5)
+    assert r.policy_version == 0
     assert r.prompt_texts == []
     assert r.response_texts == []
 
@@ -131,6 +132,7 @@ def test_rollout_generator_shapes(device):
     assert len(r.prompt_texts) == 2
     assert len(r.response_texts) == 2
     assert len(r.response_texts[0]) == 3
+    assert r.policy_version == 0
 
 
 def test_rollout_generator_uses_eval_and_restores_mode(device):
@@ -275,6 +277,26 @@ def test_rollout_runner_cache_returns_stale_flag(device):
     assert fresh2 is False
 
 
+def test_rollout_runner_tags_generation_version_and_preserves_cached_behavior(device):
+    runner, _ = _make_runner(device, rollout_interval=100)
+    batch = _make_instruction_batch(n=1)
+
+    first, first_fresh = runner(batch)
+    assert first_fresh is True
+    assert first.policy_version == 0
+
+    assert runner.update_weights(1) == 1
+    cached, cached_fresh = runner(batch)
+    assert cached is first
+    assert cached_fresh is False
+    assert cached.policy_version == 0
+
+    runner.clear_cache()
+    refreshed, refreshed_fresh = runner(batch)
+    assert refreshed_fresh is True
+    assert refreshed.policy_version == 1
+
+
 def test_rollout_runner_refreshes_for_different_batch(device):
     runner, _ = _make_runner(device, rollout_interval=100)
     r1, fresh1 = runner(_make_instruction_batch(n=1))