add server, config, and scripts

2026-06-14 19:05:18 +08:00
parent 110226d612
commit b3caa7dd7b
5 changed files with 124 additions and 0 deletions
@@ -0,0 +1,42 @@
+import os, subprocess
+from pathlib import Path
+
+def _pick_free_gpu():
+    if "CUDA_VISIBLE_DEVICES" in os.environ:
+        return os.environ["CUDA_VISIBLE_DEVICES"]
+    try:
+        out = subprocess.check_output(
+            ["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader"],
+            text=True,
+        )
+        lines = [l.strip() for l in out.strip().split("\n") if l.strip()]
+        best = max(lines, key=lambda x: int(x.split(",")[1].strip().split()[0]))
+        gpu = best.split(",")[0].strip()
+    except Exception:
+        gpu = "0"
+    os.environ["CUDA_VISIBLE_DEVICES"] = gpu
+    return gpu
+
+if __name__ == "__main__":
+    _pick_free_gpu()
+
+    model = str(Path(__file__).resolve().parent / "models/GLM-4.6V-Flash")
+
+    os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
+
+    os.execvp("vllm", [
+        "vllm", "serve",
+        model,
+        "--host", "0.0.0.0",
+        "--port", "8000",
+        "--served-model-name", "GLM-4.6V-Flash",
+        "--trust-remote-code",
+        "--enable-auto-tool-choice",
+        "--tool-call-parser", "glm47",
+        "--max-model-len", "32768",
+        "--dtype", "bfloat16",
+        "--gpu-memory-utilization", "0.85",
+        "--enforce-eager",
+        "--generation-config", "vllm",
+        "--override-generation-config", '{"temperature": 0.8, "top_k": 50, "top_p": 0.9}',
+    ])