switch to Qwen3-VL-8B-Instruct, fix start.sh arg, enable tool calling with hermes parser
This commit is contained in:
@@ -34,18 +34,18 @@ if __name__ == "__main__":
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
|
||||
os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
|
||||
|
||||
model = str(Path(__file__).resolve().parent / "models/Qwen2.5-7B-Instruct")
|
||||
model = str(Path(__file__).resolve().parent / "models/Qwen3-VL-8B-Instruct")
|
||||
|
||||
parser = make_arg_parser(FlexibleArgumentParser())
|
||||
args = parser.parse_args([
|
||||
"--model", model,
|
||||
"--host", "0.0.0.0",
|
||||
"--port", "8000",
|
||||
"--served-model-name", "Qwen2.5-7B-Instruct",
|
||||
"--served-model-name", "Qwen3-VL-8B-Instruct",
|
||||
"--trust-remote-code",
|
||||
"--max-model-len", "32768",
|
||||
"--dtype", "bfloat16",
|
||||
"--gpu-memory-utilization", "0.85",
|
||||
"--gpu-memory-utilization", "0.70",
|
||||
"--enforce-eager",
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser", "hermes",
|
||||
|
||||
Reference in New Issue
Block a user