perf: benchmark decode via real inference engine
- route decode benchmark through InferenceEngine generate path - add enable_cuda_graph toggle to engine, scheduler, and executor - make benchmark --cuda-graph/--no-cuda-graph control the toggle - hoist local time imports to module top
This commit is contained in:
@@ -188,6 +188,7 @@ class Executor:
|
||||
task_cache: TaskCacheManager,
|
||||
device: Optional[str] = None,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
enable_cuda_graph: bool = True,
|
||||
):
|
||||
self.model = model
|
||||
self.kv_cache = kv_cache
|
||||
@@ -221,7 +222,8 @@ class Executor:
|
||||
# Enabled at init-time via _warmup_cuda_graphs for CudaBackend
|
||||
# on supported head_dims; left disabled otherwise.
|
||||
self._graph_ctx = CudaGraphContext()
|
||||
self._try_enable_cuda_graph()
|
||||
if enable_cuda_graph:
|
||||
self._try_enable_cuda_graph()
|
||||
|
||||
def _try_enable_cuda_graph(self):
|
||||
if not self._graph_supported:
|
||||
|
||||
Reference in New Issue
Block a user