perf: reduce decode overhead in scheduler and executor

- Precompute page_table and decode_mask on KVCache once per step in PagePool.bind_tasks, instead of per-layer in CudaBackend/TorchNativeBackend
- Skip frequency penalty history tensor construction when all penalties are 0 in Executor.execute_decode
- Omit FrequencyPenaltyStrategy from sampling pipeline when penalty is 0
- Deduplicate get_active_tasks calls in scheduler loop (3 to 1), remove redundant sorted() on decode tasks
- Benchmark (L20, bf16, CUDA backend): B=1 9.48->9.40ms (+1%), B=4 10.73->9.89ms (+8.6%), B=8 10.77->10.13ms (+6.4%)
This commit is contained in:
2026-07-31 14:50:16 +08:00
parent 5756054d38
commit 50cfd0d555
5 changed files with 108 additions and 39 deletions
+5 -3
View File
@@ -109,9 +109,11 @@ class InferenceScheduler:
self._task_mgr.wait_for_tasks(timeout=1.0)
continue
active = self._task_mgr.get_active_tasks()
to_prefill = [
t
for t in self._task_mgr.get_active_tasks()
for t in active
if t.output_tokens == 0
and cache.task_cached(t.task_id) < len(t.prompt_ids)
]
@@ -137,10 +139,10 @@ class InferenceScheduler:
t.task_id, t.prompt_ids, start_logical_page
)
decode_tasks = self._task_mgr.get_active_tasks()
decode_tasks = active
valid: List[Task] = []
for t in sorted(decode_tasks, key=lambda t: t.task_id):
for t in decode_tasks:
if cache.task_extend(t.task_id, t.next_pos):
valid.append(t)
else: