fix: serve and train reuse the built image and expose GPUs correctly

- serve.sh/train.sh no longer pass --build on up/run; the build subcommand is the only path that rebuilds
- compose services pin image: astrai:latest so run reuses the existing image instead of triggering a rebuild
- runtime parsers leave CUDA_VISIBLE_DEVICES unset for gpu.devices: all; an empty string hid every GPU inside the container
- server service reserves count: all GPUs so CUDA_VISIBLE_DEVICES performs the only filtering, matching the trainer
- wrapper compose() strips an empty host CUDA_VISIBLE_DEVICES before invoking docker compose
This commit is contained in:
2026-08-27 12:22:02 +08:00
parent 7cfcc6c86a
commit ba8beb81be
8 changed files with 37 additions and 21 deletions
+8 -3
View File
@@ -58,7 +58,12 @@ load_config() {
}
compose() {
ASTRAI_UID="$(id -u)" ASTRAI_GID="$(id -g)" "${COMPOSE_BASE[@]}" "$@"
if [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]]; then
ASTRAI_UID="$(id -u)" ASTRAI_GID="$(id -g)" "${COMPOSE_BASE[@]}" "$@"
else
ASTRAI_UID="$(id -u)" ASTRAI_GID="$(id -g)" \
env -u CUDA_VISIBLE_DEVICES "${COMPOSE_BASE[@]}" "$@"
fi
}
checkpoint_dir() {
@@ -164,9 +169,9 @@ start_training() {
"${RUNTIME_ENV_ARGS[@]}"
)
if [[ "${foreground}" == "true" ]]; then
compose run --build --rm "${run_options[@]}" trainer "$@"
compose run --rm "${run_options[@]}" trainer "$@"
else
compose run -d --build --name "${container}" "${run_options[@]}" trainer "$@"
compose run -d --name "${container}" "${run_options[@]}" trainer "$@"
schedule_timer
log_info "Training started; run scripts/train.sh logs ${CONFIG_FILE} to follow it"
fi