fix: add tokenizer thread-safety patch and disable frontend multiprocessing for image support
This commit is contained in:
parent
3a3d0e2723
commit
b399d94de3
16
server.py
16
server.py
|
|
@ -1,8 +1,22 @@
|
||||||
import os, sys
|
import os, sys
|
||||||
import asyncio
|
import asyncio
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import threading
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import transformers
|
||||||
|
from transformers import AddedToken, PreTrainedTokenizerBase
|
||||||
|
|
||||||
|
# ── monkey-patch: make tokenizer encode thread-safe ──
|
||||||
|
_tokenizer_lock = threading.Lock()
|
||||||
|
_orig_encode = PreTrainedTokenizerBase.encode
|
||||||
|
|
||||||
|
def _safe_encode(self, *args, **kwargs):
|
||||||
|
with _tokenizer_lock:
|
||||||
|
return _orig_encode(self, *args, **kwargs)
|
||||||
|
|
||||||
|
PreTrainedTokenizerBase.encode = _safe_encode
|
||||||
|
|
||||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
from vllm.entrypoints.openai.api_server import run_server
|
from vllm.entrypoints.openai.api_server import run_server
|
||||||
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
||||||
|
|
@ -49,6 +63,8 @@ if __name__ == "__main__":
|
||||||
"--enforce-eager",
|
"--enforce-eager",
|
||||||
"--enable-auto-tool-choice",
|
"--enable-auto-tool-choice",
|
||||||
"--tool-call-parser", "hermes",
|
"--tool-call-parser", "hermes",
|
||||||
|
"--disable-frontend-multiprocessing",
|
||||||
|
"--skip-mm-profiling",
|
||||||
])
|
])
|
||||||
|
|
||||||
asyncio.run(run_server(args))
|
asyncio.run(run_server(args))
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue