diff --git a/server.py b/server.py index 7d3e6c3..818826a 100644 --- a/server.py +++ b/server.py @@ -1,8 +1,22 @@ import os, sys import asyncio import subprocess +import threading from pathlib import Path +import transformers +from transformers import AddedToken, PreTrainedTokenizerBase + +# ── monkey-patch: make tokenizer encode thread-safe ── +_tokenizer_lock = threading.Lock() +_orig_encode = PreTrainedTokenizerBase.encode + +def _safe_encode(self, *args, **kwargs): + with _tokenizer_lock: + return _orig_encode(self, *args, **kwargs) + +PreTrainedTokenizerBase.encode = _safe_encode + from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.entrypoints.openai.api_server import run_server from vllm.entrypoints.openai.cli_args import make_arg_parser @@ -49,6 +63,8 @@ if __name__ == "__main__": "--enforce-eager", "--enable-auto-tool-choice", "--tool-call-parser", "hermes", + "--disable-frontend-multiprocessing", + "--skip-mm-profiling", ]) asyncio.run(run_server(args))