fix: 修复推理引擎 batch decode 中多项正确性与并发问题
- scheduler: decode 分组由幂次分桶改为精确 next_pos,消除 KV cache 位置错乱 - task: activate() 加锁操作 active_tasks,消除数据竞争 - engine: wait_completion 加超时,防止分配失败时永久死锁 - sample: TopKStrategy 向量化为 per-sample threshold,尊重各 task 的 top_k - cache: Storage.write/gather 中 -1 页改用 mask 处理,防数据污染 - executor: prefill 逐 task 循环改为单次 tensor 调用
This commit is contained in:
@@ -59,9 +59,15 @@ class GenerateResult:
|
||||
def wait(self, timeout: Optional[float] = None) -> bool:
|
||||
return self._event.wait(timeout=timeout)
|
||||
|
||||
def wait_completion(self) -> None:
|
||||
def wait_completion(self, timeout: float = 300.0) -> None:
|
||||
with self._cond:
|
||||
self._cond.wait_for(lambda: self._completed >= self._total)
|
||||
if not self._cond.wait_for(
|
||||
lambda: self._completed >= self._total, timeout=timeout
|
||||
):
|
||||
raise TimeoutError(
|
||||
f"Generation timeout after {timeout}s "
|
||||
f"({self._completed}/{self._total} completed)"
|
||||
)
|
||||
|
||||
def get_results(self) -> List[str]:
|
||||
with self._cond:
|
||||
@@ -267,7 +273,12 @@ class InferenceEngine:
|
||||
prompts, max_tokens, temperature, top_p, top_k
|
||||
)
|
||||
|
||||
result.wait_completion()
|
||||
try:
|
||||
result.wait_completion()
|
||||
except TimeoutError:
|
||||
for tid in task_ids:
|
||||
self.scheduler.remove_task(tid)
|
||||
raise
|
||||
|
||||
for tid in task_ids:
|
||||
self.scheduler.remove_task(tid)
|
||||
|
||||
Reference in New Issue
Block a user