ViperEkura
/
ToolAgent
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: ViperEkura:9cbc7b087822e477a165ea5433e62540bc244722
Branches Tags
ViperEkura:main
..
compare: ViperEkura:b4490f26e27f13170e17246f7ef777345a16c31a
Branches Tags
ViperEkura:main

No commits in common. "9cbc7b087822e477a165ea5433e62540bc244722" and "b4490f26e27f13170e17246f7ef777345a16c31a" have entirely different histories.
9cbc7b0878 ... b4490f26e2

2 changed files with 13 additions and 31 deletions
Show Stats Expand all files Collapse all files

10
opencode.json
View File

@ -1,6 +1,5 @@
{ {
"$schema": "https://opencode.ai/config.json", "$schema": "https://opencode.ai/config.json",
"model": "Agent/Qwen3-VL-8B-Instruct",
"provider": { "provider": {
"Agent": { "Agent": {
"api": "openai", "api": "openai",
@ -9,18 +8,15 @@
"apiKey": "not-needed" "apiKey": "not-needed"
}, },
"models": { "models": {
"Qwen3-VL-8B-Instruct": { "Qwen2.5-7B-Instruct": {
"id": "Qwen3-VL-8B-Instruct", "id": "Qwen2.5-7B-Instruct",
"name": "Qwen3-VL-8B-Instruct", "name": "Qwen2.5-7B-Instruct",
"limit": { "context": 32768, "output": 8192 }, "limit": { "context": 32768, "output": 8192 },
"cost": { "input": 0, "output": 0 } "cost": { "input": 0, "output": 0 }
} }
} }
} }
}, },
"permission": {
"question": "ask"
},
"experimental": { "experimental": {
"mcp_timeout": 60000 "mcp_timeout": 60000
} }

34
server.py
View File

@ -1,18 +1,10 @@
import os, sys import os, subprocess
import asyncio
import subprocess
from pathlib import Path from pathlib import Path
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.entrypoints.openai.api_server import run_server
from vllm.entrypoints.openai.cli_args import make_arg_parser
def _pick_free_gpu(): def _pick_free_gpu():
if "CUDA_VISIBLE_DEVICES" in os.environ: if "CUDA_VISIBLE_DEVICES" in os.environ:
return os.environ["CUDA_VISIBLE_DEVICES"] return os.environ["CUDA_VISIBLE_DEVICES"]
try: try:
out = subprocess.check_output( out = subprocess.check_output(
["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader"], ["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader"],
text=True, text=True,
@ -22,33 +14,27 @@ def _pick_free_gpu():
gpu = best.split(",")[0].strip() gpu = best.split(",")[0].strip()
except Exception: except Exception:
gpu = "0" gpu = "0"
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
return gpu return gpu
if __name__ == "__main__": if __name__ == "__main__":
gpu = _pick_free_gpu() _pick_free_gpu()
if len(sys.argv) > 1:
gpu = sys.argv[1] model = str(Path(__file__).resolve().parent / "models/Qwen2.5-7B-Instruct")
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
model = str(Path(__file__).resolve().parent / "models/Qwen3-VL-8B-Instruct") os.execvp("vllm", [
"vllm", "serve",
parser = make_arg_parser(FlexibleArgumentParser()) model,
args = parser.parse_args([
"--model", model,
"--host", "0.0.0.0", "--host", "0.0.0.0",
"--port", "8000", "--port", "8000",
"--served-model-name", "Qwen3-VL-8B-Instruct", "--served-model-name", "Qwen2.5-7B-Instruct",
"--trust-remote-code", "--trust-remote-code",
"--max-model-len", "32768", "--max-model-len", "32768",
"--dtype", "bfloat16", "--dtype", "bfloat16",
"--gpu-memory-utilization", "0.70", "--gpu-memory-utilization", "0.85",
"--enforce-eager", "--enforce-eager",
"--enable-auto-tool-choice", "--enable-auto-tool-choice",
"--tool-call-parser", "hermes", "--tool-call-parser", "hermes",
]) ])
asyncio.run(run_server(args))