ViperEkura
f95722a277
- add a GPU trainer Compose profile with mounted data, models, and checkpoints - add host commands for preflight, lifecycle, logs, status, and checkpoint cleanup - resume from the latest complete checkpoint with external config or CLI arguments
86 lines
2.2 KiB
YAML
86 lines
2.2 KiB
YAML
services:
|
|
server:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
args:
|
|
CUDA_TAG: ${CUDA_TAG:-cu128}
|
|
user: "${UID:-1000}:${GID:-1000}"
|
|
ports:
|
|
- "8000:8000"
|
|
volumes:
|
|
- ./params:/app/params:ro
|
|
command: python -m scripts.tools.server --port 8000 --device cuda
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: 1
|
|
capabilities: [gpu]
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 60s
|
|
restart: unless-stopped
|
|
|
|
server-cpu:
|
|
profiles: [cpu]
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
args:
|
|
CUDA_TAG: ${CUDA_TAG:-cu128}
|
|
user: "${UID:-1000}:${GID:-1000}"
|
|
ports:
|
|
- "8000:8000"
|
|
volumes:
|
|
- ./params:/app/params:ro
|
|
command: python -m scripts.tools.server --port 8000 --device cpu
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 120s
|
|
restart: unless-stopped
|
|
|
|
trainer:
|
|
profiles: [train]
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
args:
|
|
CUDA_TAG: ${CUDA_TAG:-cu128}
|
|
init: true
|
|
user: "${UID:-1000}:${GID:-1000}"
|
|
volumes:
|
|
- ${TRAIN_DATA_DIR:-./data}:/data:ro
|
|
- ${TRAIN_MODEL_DIR:-./params}:/models/base:ro
|
|
- ${TRAIN_CHECKPOINT_DIR:-./checkpoints}:/checkpoints
|
|
environment:
|
|
- TRAIN_JOB_NAME=${TRAIN_JOB_NAME:-astrai-train}
|
|
- TRAIN_CONFIG=${TRAIN_CONFIG:-}
|
|
- BASE_MODEL=${BASE_MODEL:-/models/base}
|
|
- CHECKPOINT_ROOT=/checkpoints
|
|
- TRAIN_GPU_COUNT=${TRAIN_GPU_COUNT:-all}
|
|
- CUDA_VISIBLE_DEVICES
|
|
entrypoint: ["bash", "/app/scripts/docker/train-entrypoint.sh"]
|
|
ipc: ${TRAIN_IPC_MODE:-host}
|
|
stop_grace_period: ${TRAIN_STOP_GRACE_PERIOD:-10m}
|
|
restart: "no"
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: ${TRAIN_LOG_MAX_SIZE:-100m}
|
|
max-file: ${TRAIN_LOG_MAX_FILES:-5}
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: ${TRAIN_GPU_COUNT:-all}
|
|
capabilities: [gpu]
|