fix: resolve audited dispatch, kernel, and rollout bugs

- re-register the linear family with the operator dispatcher (ASTR_OPS / op_backend / resolve)
- fix bf16 gemv misaligned-address faults and element mispairing for offset weights
- reject misaligned bf16_swiglu inputs with a clear error and fall back in the backend gate
- make the rollout reuse decision, validation, and return atomic under one policy snapshot
- add the documented post-scoring rollout version check
- derive live+1 under the scheduler lock in optimizer_step via apply_weight_update(None, ...)
- reject rollout_max_policy_lag below rollout_interval - 1 at config time
- sync gemv stream-test inputs before switching streams; drop dead loader imports
This commit is contained in:
2026-09-03 16:58:14 +08:00
parent 736d1acb2e
commit 7e98a419a7
17 changed files with 485 additions and 49 deletions
+53 -1
View File
@@ -459,7 +459,7 @@ def test_rollout_runner_publishes_cache_before_concurrent_policy_update(device):
nonlocal validation_calls
validation_calls += 1
original_validate(result, live_version=live_version)
if validation_calls == 2:
if validation_calls == 3:
final_validation_started.set()
assert allow_final_validation_to_finish.wait(timeout=5)
@@ -503,6 +503,58 @@ def test_rollout_runner_derives_default_policy_lag_from_interval(device):
assert runner.max_policy_lag == 3
def _interleave_before_snapshot(runner, callback):
"""Wrap ``with_policy_snapshot`` so ``callback`` runs just before a
named snapshot callback enters the generator/scheduler locks."""
original_snapshot = runner.generator.with_policy_snapshot
def wrapper(inspect):
if inspect.__name__ == "reuse":
callback()
return original_snapshot(inspect)
runner.generator.with_policy_snapshot = wrapper
def test_rollout_runner_reuse_reads_cache_inside_the_snapshot(device):
"""The reuse decision must observe the cache under the policy snapshot
(regression: the cache was read outside the lock, so a concurrent
commit between the read and the lock silently handed the trainer a
stale rollout — a lost update)."""
import dataclasses
runner, _ = _make_runner(device, rollout_interval=100)
batch = _make_instruction_batch(n=1)
first, _ = runner(batch)
assert first.policy_version == 0
def concurrent_refresh():
runner.update_weights(1)
runner._cache = dataclasses.replace(first, policy_version=1)
runner._steps_since_rollout = 0
_interleave_before_snapshot(runner, concurrent_refresh)
result, fresh = runner(batch)
assert fresh is False
assert result is not first
assert result.policy_version == 1
def test_rollout_runner_recovers_when_cache_cleared_before_reuse_snapshot(device):
"""A cache clear between the reuse decision and the snapshot must
trigger a fresh rollout instead of an assertion failure (regression:
``assert cached is not None`` fired because the object was captured
outside the lock)."""
runner, _ = _make_runner(device, rollout_interval=100)
batch = _make_instruction_batch(n=1)
first, _ = runner(batch)
_interleave_before_snapshot(runner, runner.clear_cache)
result, fresh = runner(batch)
assert fresh is True
assert result is not first
@pytest.mark.parametrize(
("kwargs", "message"),
[