feat: auto-checkpoint on SIGTERM/SIGINT with DDP support
- Register SIGTERM/SIGINT handlers in training loop, set stop flag on signal - Check stop_requested at each epoch/batch boundary, break and call on_error to save checkpoint - LocalStrategy parent forwards signal to child processes via terminate(), waits up to 600s for graceful exit - TrainContext gains threading.Event-based stop_requested/request_stop - Tests verify SIGTERM/SIGINT trigger checkpoint save with exit code 0, works on both CPU and GPU
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Self
|
||||
@@ -41,6 +42,15 @@ class TrainContext:
|
||||
rank: int = field(default=0)
|
||||
kwargs: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
_stop_event: threading.Event = field(default_factory=threading.Event)
|
||||
|
||||
@property
|
||||
def stop_requested(self) -> bool:
|
||||
return self._stop_event.is_set()
|
||||
|
||||
def request_stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
|
||||
@property
|
||||
def optimizer_step(self) -> int:
|
||||
return self.consumed_samples // (
|
||||
|
||||
Reference in New Issue
Block a user