Compare commits
2 Commits
b4490f26e2
...
9cbc7b0878
| Author | SHA1 | Date |
|---|---|---|
|
|
9cbc7b0878 | |
|
|
917ff1c357 |
|
|
@ -1,5 +1,6 @@
|
||||||
{
|
{
|
||||||
"$schema": "https://opencode.ai/config.json",
|
"$schema": "https://opencode.ai/config.json",
|
||||||
|
"model": "Agent/Qwen3-VL-8B-Instruct",
|
||||||
"provider": {
|
"provider": {
|
||||||
"Agent": {
|
"Agent": {
|
||||||
"api": "openai",
|
"api": "openai",
|
||||||
|
|
@ -8,15 +9,18 @@
|
||||||
"apiKey": "not-needed"
|
"apiKey": "not-needed"
|
||||||
},
|
},
|
||||||
"models": {
|
"models": {
|
||||||
"Qwen2.5-7B-Instruct": {
|
"Qwen3-VL-8B-Instruct": {
|
||||||
"id": "Qwen2.5-7B-Instruct",
|
"id": "Qwen3-VL-8B-Instruct",
|
||||||
"name": "Qwen2.5-7B-Instruct",
|
"name": "Qwen3-VL-8B-Instruct",
|
||||||
"limit": { "context": 32768, "output": 8192 },
|
"limit": { "context": 32768, "output": 8192 },
|
||||||
"cost": { "input": 0, "output": 0 }
|
"cost": { "input": 0, "output": 0 }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"permission": {
|
||||||
|
"question": "ask"
|
||||||
|
},
|
||||||
"experimental": {
|
"experimental": {
|
||||||
"mcp_timeout": 60000
|
"mcp_timeout": 60000
|
||||||
}
|
}
|
||||||
|
|
|
||||||
34
server.py
34
server.py
|
|
@ -1,10 +1,18 @@
|
||||||
import os, subprocess
|
import os, sys
|
||||||
|
import asyncio
|
||||||
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
from vllm.entrypoints.openai.api_server import run_server
|
||||||
|
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||||
|
|
||||||
|
|
||||||
def _pick_free_gpu():
|
def _pick_free_gpu():
|
||||||
if "CUDA_VISIBLE_DEVICES" in os.environ:
|
if "CUDA_VISIBLE_DEVICES" in os.environ:
|
||||||
return os.environ["CUDA_VISIBLE_DEVICES"]
|
return os.environ["CUDA_VISIBLE_DEVICES"]
|
||||||
try:
|
try:
|
||||||
|
|
||||||
out = subprocess.check_output(
|
out = subprocess.check_output(
|
||||||
["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader"],
|
["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader"],
|
||||||
text=True,
|
text=True,
|
||||||
|
|
@ -14,27 +22,33 @@ def _pick_free_gpu():
|
||||||
gpu = best.split(",")[0].strip()
|
gpu = best.split(",")[0].strip()
|
||||||
except Exception:
|
except Exception:
|
||||||
gpu = "0"
|
gpu = "0"
|
||||||
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
|
|
||||||
return gpu
|
return gpu
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
_pick_free_gpu()
|
gpu = _pick_free_gpu()
|
||||||
|
if len(sys.argv) > 1:
|
||||||
model = str(Path(__file__).resolve().parent / "models/Qwen2.5-7B-Instruct")
|
gpu = sys.argv[1]
|
||||||
|
|
||||||
|
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
|
||||||
os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
|
os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
|
||||||
|
|
||||||
os.execvp("vllm", [
|
model = str(Path(__file__).resolve().parent / "models/Qwen3-VL-8B-Instruct")
|
||||||
"vllm", "serve",
|
|
||||||
model,
|
parser = make_arg_parser(FlexibleArgumentParser())
|
||||||
|
args = parser.parse_args([
|
||||||
|
"--model", model,
|
||||||
"--host", "0.0.0.0",
|
"--host", "0.0.0.0",
|
||||||
"--port", "8000",
|
"--port", "8000",
|
||||||
"--served-model-name", "Qwen2.5-7B-Instruct",
|
"--served-model-name", "Qwen3-VL-8B-Instruct",
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
"--max-model-len", "32768",
|
"--max-model-len", "32768",
|
||||||
"--dtype", "bfloat16",
|
"--dtype", "bfloat16",
|
||||||
"--gpu-memory-utilization", "0.85",
|
"--gpu-memory-utilization", "0.70",
|
||||||
"--enforce-eager",
|
"--enforce-eager",
|
||||||
"--enable-auto-tool-choice",
|
"--enable-auto-tool-choice",
|
||||||
"--tool-call-parser", "hermes",
|
"--tool-call-parser", "hermes",
|
||||||
])
|
])
|
||||||
|
|
||||||
|
asyncio.run(run_server(args))
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue