feat: add persistent DataLoader workers

- Keep training workers alive between epochs when enabled.
- Avoid invalid prefetch settings for single-process loading.

scripts/tools/train.py

@@ -251,6 +251,12 @@ _START_METHODS = ["spawn", "fork", "forkserver"]
     group="Data Loading",
     help="Pin memory.",
 )
+@opt(
+    "--persistent_workers/--no-persistent_workers",
+    default=True,
+    group="Data Loading",
+    help="Keep DataLoader workers alive between epochs.",
+)
 @opt(
     "--window_size",
     type=int,
@@ -606,6 +612,7 @@ def train(
     random_seed: int,
     num_workers: int,
     pin_memory: bool,
+    persistent_workers: bool,
     gradient_checkpointing: bool,
     window_size: int,
     stride: int,
@@ -797,6 +804,7 @@ def train(
         random_seed=random_seed,
         num_workers=num_workers,
         pin_memory=pin_memory,
+        persistent_workers=persistent_workers,
         nprocs=nprocs,
         backend=backend,
         master_addr=master_addr,