feat: unify Docker serving configuration in YAML
- Add server.py --config serve.yaml; explicit CLI flags override YAML - Add scripts/serve.sh and serve_runtime.py for the Compose lifecycle - Template server/cpu ports and param mounts in docker-compose.yml - Document schema in docs/developer/docker-serving.md and params guide - Add tests for runtime parsing and server CLI merge logic
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
"""Parse the host-side runtime section of a serving configuration.
|
||||
|
||||
The Compose wrapper needs a few container-side values on the host as well:
|
||||
``server.port`` (the port the container listens on) and ``server.device``
|
||||
(used by the preflight GPU consistency check). Everything else under
|
||||
``server:`` is owned by ``scripts/tools/server.py --config`` inside the
|
||||
container.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ENV_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
def _mapping(value, name: str) -> dict:
|
||||
if value is None:
|
||||
return {}
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"{name} must be a mapping")
|
||||
return value
|
||||
|
||||
|
||||
def _path(value, name: str, config_dir: Path) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ValueError(f"runtime.paths.{name} is required")
|
||||
path = Path(value).expanduser()
|
||||
if not path.is_absolute():
|
||||
path = config_dir / path
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def _port(value, name: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ValueError(f"{name} must be an integer")
|
||||
if not 1 <= value <= 65535:
|
||||
raise ValueError(f"{name} must be between 1 and 65535")
|
||||
return value
|
||||
|
||||
|
||||
def load_runtime(config_path: str) -> dict[str, str]:
|
||||
path = Path(config_path).resolve()
|
||||
with path.open(encoding="utf-8") as file:
|
||||
config = yaml.safe_load(file) or {}
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError("serving configuration must be a mapping")
|
||||
|
||||
runtime = _mapping(config.get("runtime"), "runtime")
|
||||
if not runtime:
|
||||
raise ValueError("top-level runtime section is required")
|
||||
paths = _mapping(runtime.get("paths"), "paths")
|
||||
gpu = _mapping(runtime.get("gpu"), "gpu")
|
||||
container = _mapping(runtime.get("container"), "container")
|
||||
environment = _mapping(runtime.get("environment"), "environment")
|
||||
server = _mapping(config.get("server"), "server")
|
||||
|
||||
job_name = runtime.get("job_name", "")
|
||||
if job_name and not isinstance(job_name, str):
|
||||
raise ValueError("runtime.job_name must be a string")
|
||||
if job_name and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", job_name):
|
||||
raise ValueError(
|
||||
"runtime.job_name must use letters, numbers, dot, underscore, or dash"
|
||||
)
|
||||
|
||||
port = _port(runtime.get("port", 8000), "runtime.port")
|
||||
container_port = _port(server.get("port", 8000), "server.port")
|
||||
|
||||
device = server.get("device", "cuda")
|
||||
if not isinstance(device, str) or not device.strip():
|
||||
raise ValueError("server.device must be a string")
|
||||
|
||||
gpu_enabled = gpu.get("enabled", True)
|
||||
if not isinstance(gpu_enabled, bool):
|
||||
raise ValueError("runtime.gpu.enabled must be a boolean")
|
||||
|
||||
devices = gpu.get("devices", "all")
|
||||
if gpu_enabled:
|
||||
if devices == "all":
|
||||
visible_devices = ""
|
||||
elif isinstance(devices, list) and len(devices) == 1:
|
||||
text = str(devices[0])
|
||||
if not text.isdigit():
|
||||
raise ValueError(
|
||||
"runtime.gpu.devices entries must be non-negative integers"
|
||||
)
|
||||
visible_devices = text
|
||||
else:
|
||||
raise ValueError(
|
||||
"runtime.gpu.devices must be 'all' or a single-device list such as [0]"
|
||||
)
|
||||
else:
|
||||
visible_devices = ""
|
||||
if devices != "all":
|
||||
raise ValueError(
|
||||
"runtime.gpu.devices is ignored when runtime.gpu.enabled is false"
|
||||
)
|
||||
if device != "cpu":
|
||||
raise ValueError(
|
||||
"server.device must be 'cpu' when runtime.gpu.enabled is false"
|
||||
)
|
||||
|
||||
values = {
|
||||
"SERVE_JOB_NAME": job_name,
|
||||
"SERVE_PORT": str(port),
|
||||
"SERVE_CONTAINER_PORT": str(container_port),
|
||||
"SERVE_PARAM_DIR": _path(paths.get("param", "./params"), "param", path.parent),
|
||||
"SERVE_GPU_ENABLED": "true" if gpu_enabled else "false",
|
||||
"SERVE_DEVICE": device,
|
||||
"CUDA_VISIBLE_DEVICES": visible_devices,
|
||||
"CUDA_TAG": str(container.get("cuda_tag", "cu128")),
|
||||
}
|
||||
|
||||
for name, value in environment.items():
|
||||
if not isinstance(name, str) or not ENV_NAME.fullmatch(name):
|
||||
raise ValueError(f"invalid runtime.environment name: {name!r}")
|
||||
if value is not None and not isinstance(value, (str, int, float, bool)):
|
||||
raise ValueError(f"runtime.environment.{name} must be a scalar")
|
||||
values["environment"] = environment
|
||||
return values
|
||||
|
||||
|
||||
def shell_exports(runtime: dict[str, str]) -> str:
|
||||
return "\n".join(
|
||||
f"export {name}={shlex.quote(value)}"
|
||||
for name, value in runtime.items()
|
||||
if name != "environment"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("command", choices=("exports", "environment"))
|
||||
parser.add_argument("config")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
runtime = load_runtime(args.config)
|
||||
except (OSError, ValueError, yaml.YAMLError) as exc:
|
||||
parser.error(str(exc))
|
||||
|
||||
if args.command == "exports":
|
||||
print(shell_exports(runtime))
|
||||
return
|
||||
for name, value in runtime["environment"].items():
|
||||
rendered = "" if value is None else str(value)
|
||||
print(f"{name}={rendered}", end="\0")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+128
-2
@@ -2,16 +2,105 @@ from pathlib import Path
|
||||
|
||||
import click
|
||||
import torch
|
||||
import yaml
|
||||
from click.core import ParameterSource
|
||||
|
||||
from astrai.inference import run_server
|
||||
|
||||
_DTYPES = ["bfloat16", "float16", "float32"]
|
||||
_SERVER_KEYS = (
|
||||
"host",
|
||||
"port",
|
||||
"reload",
|
||||
"param_path",
|
||||
"device",
|
||||
"dtype",
|
||||
"max_batch_size",
|
||||
"max_seq_len",
|
||||
)
|
||||
|
||||
|
||||
def _merge_yaml_into_kwargs(
|
||||
config_path: str,
|
||||
passed_kwargs: dict,
|
||||
explicit_keys: set[str] | None = None,
|
||||
) -> dict:
|
||||
"""Merge Click defaults, YAML server values, then explicit CLI values."""
|
||||
with open(config_path, encoding="utf-8") as file:
|
||||
config = yaml.safe_load(file) or {}
|
||||
if not isinstance(config, dict):
|
||||
raise click.UsageError(f"Serving config must be a mapping: {config_path}")
|
||||
server = config.get("server") or {}
|
||||
if not isinstance(server, dict):
|
||||
raise click.UsageError("top-level server section must be a mapping")
|
||||
|
||||
unknown = sorted(set(server) - set(_SERVER_KEYS))
|
||||
if unknown:
|
||||
click.echo(
|
||||
f"Warning: ignoring unknown server config keys: {', '.join(unknown)}",
|
||||
err=True,
|
||||
)
|
||||
|
||||
merged = dict(passed_kwargs)
|
||||
merged.update({key: server[key] for key in _SERVER_KEYS if key in server})
|
||||
if explicit_keys is None:
|
||||
explicit_keys = set(passed_kwargs)
|
||||
for key in explicit_keys:
|
||||
if key in passed_kwargs:
|
||||
merged[key] = passed_kwargs[key]
|
||||
return merged
|
||||
|
||||
|
||||
def _as_int(value, name: str) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
raise click.UsageError(f"{name} must be an integer")
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
raise click.UsageError(f"{name} must be an integer, got {value!r}") from None
|
||||
|
||||
|
||||
def _resolve_server_config(
|
||||
config_path: str,
|
||||
passed_kwargs: dict,
|
||||
explicit_keys: set[str] | None = None,
|
||||
) -> dict:
|
||||
"""Merge YAML values, then coerce and validate the resolved settings.
|
||||
|
||||
``explicit_keys`` are CLI flags that win over YAML; when None, YAML values
|
||||
win over Click defaults.
|
||||
"""
|
||||
merged = _merge_yaml_into_kwargs(config_path, passed_kwargs, explicit_keys or set())
|
||||
resolved = dict(merged)
|
||||
resolved["port"] = _as_int(resolved["port"], "server.port") or 8000
|
||||
resolved["max_batch_size"] = (
|
||||
_as_int(resolved["max_batch_size"], "server.max_batch_size") or 16
|
||||
)
|
||||
resolved["max_seq_len"] = _as_int(resolved["max_seq_len"], "server.max_seq_len")
|
||||
resolved["reload"] = bool(resolved["reload"])
|
||||
if resolved["dtype"] not in _DTYPES:
|
||||
raise click.UsageError(
|
||||
f"server.dtype must be one of {', '.join(_DTYPES)}, got {resolved['dtype']!r}"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
@click.command(name="serve", help="Launch inference server (OpenAI-compatible API).")
|
||||
@click.option(
|
||||
"--config",
|
||||
"-c",
|
||||
"config_path",
|
||||
type=click.Path(exists=True, dir_okay=False),
|
||||
default=None,
|
||||
help="Serving YAML config. CLI flags override YAML values.",
|
||||
)
|
||||
@click.option("--host", default="0.0.0.0", help="Host address.")
|
||||
@click.option("--port", type=int, default=8000, help="Port number.")
|
||||
@click.option("--reload", is_flag=True, help="Enable auto-reload for development.")
|
||||
@click.option(
|
||||
"--reload", is_flag=True, default=False, help="Enable auto-reload for development."
|
||||
)
|
||||
@click.option(
|
||||
"--param_path",
|
||||
type=click.Path(exists=True),
|
||||
@@ -37,10 +126,47 @@ _DTYPES = ["bfloat16", "float16", "float32"]
|
||||
default=None,
|
||||
help="Maximum sequence length (KV cache size + prompt truncation). Uses model config if not set.",
|
||||
)
|
||||
@click.pass_context
|
||||
def server_command(
|
||||
host, port, reload, param_path, device, dtype, max_batch_size, max_seq_len
|
||||
ctx,
|
||||
config_path,
|
||||
host,
|
||||
port,
|
||||
reload,
|
||||
param_path,
|
||||
device,
|
||||
dtype,
|
||||
max_batch_size,
|
||||
max_seq_len,
|
||||
):
|
||||
"""Launch inference server (OpenAI-compatible API)."""
|
||||
if config_path:
|
||||
passed_kwargs = {
|
||||
"host": host,
|
||||
"port": port,
|
||||
"reload": reload,
|
||||
"param_path": param_path,
|
||||
"device": device,
|
||||
"dtype": dtype,
|
||||
"max_batch_size": max_batch_size,
|
||||
"max_seq_len": max_seq_len,
|
||||
}
|
||||
explicit_keys = {
|
||||
key
|
||||
for key in passed_kwargs
|
||||
if ctx.get_parameter_source(key) is ParameterSource.COMMANDLINE
|
||||
}
|
||||
resolved = _resolve_server_config(config_path, passed_kwargs, explicit_keys)
|
||||
host = resolved["host"]
|
||||
port = resolved["port"]
|
||||
reload = resolved["reload"]
|
||||
param_path = resolved["param_path"]
|
||||
device = resolved["device"]
|
||||
dtype = resolved["dtype"]
|
||||
max_batch_size = resolved["max_batch_size"]
|
||||
max_seq_len = resolved["max_seq_len"]
|
||||
click.echo(f"Config: {config_path}")
|
||||
|
||||
dtype_map = {
|
||||
"bfloat16": torch.bfloat16,
|
||||
"float16": torch.float16,
|
||||
|
||||
Reference in New Issue
Block a user