perf: benchmark decode via real inference engine

- route decode benchmark through InferenceEngine generate path
- add enable_cuda_graph toggle to engine, scheduler, and executor
- make benchmark --cuda-graph/--no-cuda-graph control the toggle
- hoist local time imports to module top
This commit is contained in:
2026-08-09 11:47:14 +08:00
parent cf4f5ab9f6
commit c1d05ae11d
4 changed files with 72 additions and 16 deletions
+3 -1
View File
@@ -188,6 +188,7 @@ class Executor:
task_cache: TaskCacheManager,
device: Optional[str] = None,
dtype: Optional[torch.dtype] = None,
enable_cuda_graph: bool = True,
):
self.model = model
self.kv_cache = kv_cache
@@ -221,7 +222,8 @@ class Executor:
# Enabled at init-time via _warmup_cuda_graphs for CudaBackend
# on supported head_dims; left disabled otherwise.
self._graph_ctx = CudaGraphContext()
self._try_enable_cuda_graph()
if enable_cuda_graph:
self._try_enable_cuda_graph()
def _try_enable_cuda_graph(self):
if not self._graph_supported: