perf: batch decode stream callbacks into one dispatch per step

- add BatchedStreamCallback sink type: TaskManager resolves a decode step's (task_id, token) events under one lock and delivers each sink a single list instead of one call per token
- keep the plain Callable[[str]] callback contract: per-token callbacks still receive one call per event, and invoke_callback/cancel_task wrap single events for batched sinks
- collect aborted, text, and finish STOP events in the scheduler decode loop and dispatch once per step instead of once per token
- register one _ResultSink per generate call (replacing per-task closures) so GenerateResult takes its lock and wakes waiters once per step, with late-bind replay for tasks that start decoding before add_task returns their id
- apply GenerateResult batches under a single condition hold via append_batch; append delegates to it
- update engine test fakes to the batched contract and add coverage for event grouping, single-event dispatch, cancel STOP, and late-bind replay

Benchmark: NVIDIA L20 (idle), CUDA 12.8, torch 2.11.0+cu128, 1.2B bf16 checkpoint, prompt 512, 256 greedy tokens, CUDA graph on, serving-level decode, 3 trials
- batch 32: 7.808 -> 7.506 ms/token (4098 -> 4263 batch tok/s, +4.0%)
- batch 1/8: unchanged within noise (3.768 -> 3.797 / 4.699 -> 4.607 ms/token)
- full suite: 896 passed
This commit is contained in:
2026-09-05 00:03:36 +08:00
parent 1798474316
commit 074642b6d2
6 changed files with 265 additions and 46 deletions
+60 -1
View File
@@ -4,7 +4,23 @@ from unittest.mock import MagicMock
import pytest
from astrai.inference import STOP, Task, TaskManager, TaskStatus
from astrai.inference import (
STOP,
BatchedStreamCallback,
Task,
TaskManager,
TaskStatus,
)
class RecordingSink(BatchedStreamCallback):
"""Batch-aware callback capturing every dispatch as one batch."""
def __init__(self):
self.batches = []
def __call__(self, events):
self.batches.append(events)
def _make_mock_tokenizer():
@@ -217,3 +233,46 @@ def test_task_manager_cancel_active_task_delivers_stop_callback():
immediate, cancelled = tm.cancel_task(task_id)
assert cancelled and immediate == []
assert received == [STOP]
def test_invoke_callbacks_batches_sink_events_and_keeps_plain_per_token():
tm = TaskManager(tokenizer=_make_mock_tokenizer())
plain = []
tid_plain = tm.add_task("plain", stream_callback=plain.append)
sink = RecordingSink()
tid_a = tm.add_task("sink a", stream_callback=sink)
tid_b = tm.add_task("sink b", stream_callback=sink)
tm.invoke_callbacks(
[
(tid_a, "x"),
(tid_plain, "p"),
(tid_b, "y"),
("unknown-task", "dropped"),
(tid_a, STOP),
]
)
assert plain == ["p"]
assert sink.batches == [[(tid_a, "x"), (tid_b, "y"), (tid_a, STOP)]]
def test_invoke_callback_delivers_single_event_to_batched_sink():
tm = TaskManager(tokenizer=_make_mock_tokenizer())
sink = RecordingSink()
task_id = tm.add_task("test", stream_callback=sink)
tm.invoke_callback(task_id, STOP)
assert sink.batches == [[(task_id, STOP)]]
def test_cancel_delivers_batched_stop_to_sink():
tm = TaskManager(tokenizer=_make_mock_tokenizer())
sink = RecordingSink()
task_id = tm.add_task("test", stream_callback=sink)
immediate, cancelled = tm.cancel_task(task_id)
assert cancelled
assert sink.batches == [[(task_id, STOP)]]