perf: benchmark decode via real inference engine

- route decode benchmark through InferenceEngine generate path
- add enable_cuda_graph toggle to engine, scheduler, and executor
- make benchmark --cuda-graph/--no-cuda-graph control the toggle
- hoist local time imports to module top
This commit is contained in:
2026-08-09 11:47:14 +08:00
parent cf4f5ab9f6
commit c1d05ae11d
4 changed files with 72 additions and 16 deletions
+2
View File
@@ -74,6 +74,7 @@ class InferenceEngine:
max_batch_size: int = 1,
max_seq_len: Optional[int] = None,
cache: Optional[PagePool] = None,
enable_cuda_graph: bool = True,
):
self.model = model
self.tokenizer = tokenizer
@@ -83,6 +84,7 @@ class InferenceEngine:
max_batch_size=max_batch_size,
max_seq_len=max_seq_len,
cache=cache,
enable_cuda_graph=enable_cuda_graph,
)
self.scheduler.start()