fix: add tokenizer thread-safety patch and disable frontend multiprocessing for image support
This commit is contained in:
parent
3a3d0e2723
commit
b399d94de3
16
server.py
16
server.py
|
|
@ -1,8 +1,22 @@
|
|||
import os, sys
|
||||
import asyncio
|
||||
import subprocess
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import transformers
|
||||
from transformers import AddedToken, PreTrainedTokenizerBase
|
||||
|
||||
# ── monkey-patch: make tokenizer encode thread-safe ──
|
||||
_tokenizer_lock = threading.Lock()
|
||||
_orig_encode = PreTrainedTokenizerBase.encode
|
||||
|
||||
def _safe_encode(self, *args, **kwargs):
|
||||
with _tokenizer_lock:
|
||||
return _orig_encode(self, *args, **kwargs)
|
||||
|
||||
PreTrainedTokenizerBase.encode = _safe_encode
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.entrypoints.openai.api_server import run_server
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||
|
|
@ -49,6 +63,8 @@ if __name__ == "__main__":
|
|||
"--enforce-eager",
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser", "hermes",
|
||||
"--disable-frontend-multiprocessing",
|
||||
"--skip-mm-profiling",
|
||||
])
|
||||
|
||||
asyncio.run(run_server(args))
|
||||
|
|
|
|||
Loading…
Reference in New Issue