ViperEkura
71b6e3aaaf
- rewrite docker.sh with gpu default and --no-gpu override - inject host uid/gid via ASTRAI_UID/GID in train.sh compose() - filter readonly UID/GID lines when sourcing .env.train - build image user via USER_UID/USER_GID args matching host uid/gid - pass all GPUs (count: all) and filter by CUDA_VISIBLE_DEVICES inside the container - forward NCCL vars through compose environment
94 lines
2.5 KiB
YAML
94 lines
2.5 KiB
YAML
services:
|
|
server:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
args:
|
|
CUDA_TAG: ${CUDA_TAG:-cu128}
|
|
USER_UID: ${ASTRAI_UID:-1000}
|
|
USER_GID: ${ASTRAI_GID:-1000}
|
|
user: "${ASTRAI_UID:-1000}:${ASTRAI_GID:-1000}"
|
|
ports:
|
|
- "8000:8000"
|
|
volumes:
|
|
- ./params:/app/params:ro
|
|
command: python -m scripts.tools.server --port 8000 --device cuda
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: 1
|
|
capabilities: [gpu]
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 60s
|
|
restart: unless-stopped
|
|
|
|
server-cpu:
|
|
profiles: [cpu]
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
args:
|
|
CUDA_TAG: ${CUDA_TAG:-cu128}
|
|
USER_UID: ${ASTRAI_UID:-1000}
|
|
USER_GID: ${ASTRAI_GID:-1000}
|
|
user: "${ASTRAI_UID:-1000}:${ASTRAI_GID:-1000}"
|
|
ports:
|
|
- "8000:8000"
|
|
volumes:
|
|
- ./params:/app/params:ro
|
|
command: python -m scripts.tools.server --port 8000 --device cpu
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 120s
|
|
restart: unless-stopped
|
|
|
|
trainer:
|
|
profiles: [train]
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
args:
|
|
CUDA_TAG: ${CUDA_TAG:-cu128}
|
|
USER_UID: ${ASTRAI_UID:-1000}
|
|
USER_GID: ${ASTRAI_GID:-1000}
|
|
init: true
|
|
user: "${ASTRAI_UID:-1000}:${ASTRAI_GID:-1000}"
|
|
volumes:
|
|
- ${TRAIN_DATA_DIR:-./data}:/data:ro
|
|
- ${TRAIN_MODEL_DIR:-./params}:/models/base:ro
|
|
- ${TRAIN_CHECKPOINT_DIR:-./checkpoints}:/checkpoints
|
|
environment:
|
|
- TRAIN_JOB_NAME=${TRAIN_JOB_NAME:-astrai-train}
|
|
- TRAIN_CONFIG=${TRAIN_CONFIG:-}
|
|
- BASE_MODEL=${BASE_MODEL:-/models/base}
|
|
- CHECKPOINT_ROOT=/checkpoints
|
|
- TRAIN_GPU_COUNT=${TRAIN_GPU_COUNT:-all}
|
|
- CUDA_VISIBLE_DEVICES
|
|
- NCCL_P2P_DISABLE
|
|
- NCCL_NET_GDR_LEVEL
|
|
entrypoint: ["bash", "/app/scripts/docker/train-entrypoint.sh"]
|
|
ipc: ${TRAIN_IPC_MODE:-host}
|
|
stop_grace_period: ${TRAIN_STOP_GRACE_PERIOD:-10m}
|
|
restart: "no"
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: ${TRAIN_LOG_MAX_SIZE:-100m}
|
|
max-file: ${TRAIN_LOG_MAX_FILES:-5}
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|