fix: prevent signal handler test from racing with training completion
- Set n_epoch=99999 so training runs until parent delivers signal instead of finishing too fast on CPU - Drop ready-file deadline from 30s to 10s
This commit is contained in:
@@ -71,7 +71,7 @@ def _inner_run(batch_per_device, ckpt_interval, ckpt_dir, ready_file):
|
|||||||
optimizer_fn=optimizer_fn,
|
optimizer_fn=optimizer_fn,
|
||||||
scheduler_fn=scheduler_fn,
|
scheduler_fn=scheduler_fn,
|
||||||
ckpt_dir=ckpt_dir,
|
ckpt_dir=ckpt_dir,
|
||||||
n_epoch=1,
|
n_epoch=99999,
|
||||||
batch_per_device=batch_per_device,
|
batch_per_device=batch_per_device,
|
||||||
ckpt_interval=ckpt_interval,
|
ckpt_interval=ckpt_interval,
|
||||||
grad_accum_steps=1,
|
grad_accum_steps=1,
|
||||||
@@ -94,7 +94,7 @@ def _spawn_train_and_signal(ckpt_dir, sig, timeout=120):
|
|||||||
)
|
)
|
||||||
p.start()
|
p.start()
|
||||||
|
|
||||||
deadline = time.time() + 30
|
deadline = time.time() + 10
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
if os.path.exists(ready_file):
|
if os.path.exists(ready_file):
|
||||||
with open(ready_file) as f:
|
with open(ready_file) as f:
|
||||||
|
|||||||
Reference in New Issue
Block a user