fix: serve and train reuse the built image and expose GPUs correctly

- serve.sh/train.sh no longer pass --build on up/run; the build subcommand is the only path that rebuilds
- compose services pin image: astrai:latest so run reuses the existing image instead of triggering a rebuild
- runtime parsers leave CUDA_VISIBLE_DEVICES unset for gpu.devices: all; an empty string hid every GPU inside the container
- server service reserves count: all GPUs so CUDA_VISIBLE_DEVICES performs the only filtering, matching the trainer
- wrapper compose() strips an empty host CUDA_VISIBLE_DEVICES before invoking docker compose
This commit is contained in:
2026-08-27 12:22:02 +08:00
parent 7cfcc6c86a
commit ba8beb81be
8 changed files with 37 additions and 21 deletions
+8 -3
View File
@@ -55,7 +55,12 @@ load_config() {
}
compose() {
ASTRAI_UID="$(id -u)" ASTRAI_GID="$(id -g)" "${COMPOSE_BASE[@]}" "$@"
if [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]]; then
ASTRAI_UID="$(id -u)" ASTRAI_GID="$(id -g)" "${COMPOSE_BASE[@]}" "$@"
else
ASTRAI_UID="$(id -u)" ASTRAI_GID="$(id -g)" \
env -u CUDA_VISIBLE_DEVICES "${COMPOSE_BASE[@]}" "$@"
fi
}
container_name() {
@@ -129,11 +134,11 @@ start_server() {
"${RUNTIME_ENV_ARGS[@]}"
)
if [[ "${foreground}" == "true" ]]; then
compose "${PROFILE_ARGS[@]}" run --build --rm --service-ports \
compose "${PROFILE_ARGS[@]}" run --rm --service-ports \
"${run_options[@]}" "$(service_name)" \
python -m scripts.tools.server --config /run/astrai/serve.yaml "$@"
else
compose "${PROFILE_ARGS[@]}" run -d --build --service-ports \
compose "${PROFILE_ARGS[@]}" run -d --service-ports \
--name "${container}" "${run_options[@]}" "$(service_name)" \
python -m scripts.tools.server --config /run/astrai/serve.yaml "$@"
log_info "Server started; run scripts/serve.sh logs ${CONFIG_FILE} to follow it"