diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..225ef29 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# Model files (large, downloaded separately) +models/ + +# log +*.log \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..60bc3fe --- /dev/null +++ b/install.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +pip install torch==2.10.0 torchvision==0.25.0 \ + --index-url https://download.pytorch.org/whl/cu128 +pip install vllm==0.18.0 flashinfer-python==0.6.6 +pip install transformers==5.12.0 diff --git a/opencode.json b/opencode.json new file mode 100644 index 0000000..9b76881 --- /dev/null +++ b/opencode.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://opencode.ai/config.json", + "provider": { + "Agent": { + "api": "openai", + "options": { + "baseURL": "http://localhost:8000/v1/", + "apiKey": "not-needed" + }, + "models": { + "GLM-4.6V-Flash": { + "id": "GLM-4.6V-Flash", + "name": "GLM-4.6V-Flash", + "limit": { "context": 32768, "output": 8192 }, + "cost": { "input": 0, "output": 0 }, + "modalities": { "input": ["text", "image"], "output": ["text"] } + } + } + } + }, + "experimental": { + "mcp_timeout": 60000 + } +} diff --git a/server.py b/server.py new file mode 100644 index 0000000..baa5de3 --- /dev/null +++ b/server.py @@ -0,0 +1,42 @@ +import os, subprocess +from pathlib import Path + +def _pick_free_gpu(): + if "CUDA_VISIBLE_DEVICES" in os.environ: + return os.environ["CUDA_VISIBLE_DEVICES"] + try: + out = subprocess.check_output( + ["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader"], + text=True, + ) + lines = [l.strip() for l in out.strip().split("\n") if l.strip()] + best = max(lines, key=lambda x: int(x.split(",")[1].strip().split()[0])) + gpu = best.split(",")[0].strip() + except Exception: + gpu = "0" + os.environ["CUDA_VISIBLE_DEVICES"] = gpu + return gpu + +if __name__ == "__main__": + _pick_free_gpu() + + model = str(Path(__file__).resolve().parent / "models/GLM-4.6V-Flash") + + os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" + + os.execvp("vllm", [ + "vllm", "serve", + model, + "--host", "0.0.0.0", + "--port", "8000", + "--served-model-name", "GLM-4.6V-Flash", + "--trust-remote-code", + "--enable-auto-tool-choice", + "--tool-call-parser", "glm47", + "--max-model-len", "32768", + "--dtype", "bfloat16", + "--gpu-memory-utilization", "0.85", + "--enforce-eager", + "--generation-config", "vllm", + "--override-generation-config", '{"temperature": 0.8, "top_k": 50, "top_p": 0.9}', + ]) diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..4e1c357 --- /dev/null +++ b/start.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +ACTION="${1:-all}" +GPU="${2:-2}" +PROJECT_DIR="$(cd "$(dirname "$0")" && pwd)" + +start_backend() { + echo "=== Starting Backend (vLLM) on GPU $GPU ===" + source /home/kxqandccx/miniconda3/bin/activate agent_use + CUDA_VISIBLE_DEVICES="$GPU" nohup python "$PROJECT_DIR/server.py" \ + > "$PROJECT_DIR/server.log" 2>&1 & + echo "Backend PID=$!" + + echo "Waiting for backend..." + for i in $(seq 1 60); do + if curl -s http://127.0.0.1:8000/v1/models > /dev/null 2>&1; then + echo "Backend ready!" + return + fi + sleep 2 + done + echo "Backend failed to start" + exit 1 +} + +start_frontend() { + echo "=== Starting Frontend (opencode web) ===" + cd "$PROJECT_DIR" + nohup opencode web --port 3000 > /tmp/opencode-web.log 2>&1 & + echo "Frontend PID=$!" + echo "Web interface: http://127.0.0.1:3000/" +} + +case "$ACTION" in + backend) start_backend ;; + frontend) start_frontend ;; + all) start_backend; start_frontend ;; + *) + echo "Usage: $0 [backend|frontend|all] [GPU]" + echo " ./start.sh # start both" + echo " ./start.sh backend # start backend only" + echo " ./start.sh frontend # start frontend only" + echo " ./start.sh all 3 # both on GPU 3" + ;; +esac