perf: benchmark decode via real inference engine

- route decode benchmark through InferenceEngine generate path
- add enable_cuda_graph toggle to engine, scheduler, and executor
- make benchmark --cuda-graph/--no-cuda-graph control the toggle
- hoist local time imports to module top
This commit is contained in:
2026-08-09 11:47:14 +08:00
parent cf4f5ab9f6
commit c1d05ae11d
4 changed files with 72 additions and 16 deletions
+2
View File
@@ -27,6 +27,7 @@ class InferenceScheduler:
device: Optional[str] = None,
dtype: Optional[torch.dtype] = None,
cache: Optional[PagePool] = None,
enable_cuda_graph: bool = True,
):
config = model.config
@@ -74,6 +75,7 @@ class InferenceScheduler:
task_cache=self._task_cache,
device=self.device,
dtype=self.dtype,
enable_cuda_graph=enable_cuda_graph,
)
self._stop_event = threading.Event()