fix: 修复推理引擎 batch decode 中多项正确性与并发问题
- scheduler: decode 分组由幂次分桶改为精确 next_pos,消除 KV cache 位置错乱 - task: activate() 加锁操作 active_tasks,消除数据竞争 - engine: wait_completion 加超时,防止分配失败时永久死锁 - sample: TopKStrategy 向量化为 per-sample threshold,尊重各 task 的 top_k - cache: Storage.write/gather 中 -1 页改用 mask 处理,防数据污染 - executor: prefill 逐 task 循环改为单次 tensor 调用
This commit is contained in:
@@ -64,16 +64,26 @@ class TopKStrategy(BaseSamplingStrategy):
|
||||
def apply(self, logits, filter_value=-float("inf")):
|
||||
tk = self.top_k
|
||||
if isinstance(tk, Tensor):
|
||||
tk = tk.to(logits.device, non_blocking=True).long().clamp(min=0)
|
||||
max_k = int(tk.max().item())
|
||||
if max_k <= 0:
|
||||
return logits
|
||||
k = min(max_k, logits.size(-1))
|
||||
elif tk > 0:
|
||||
k = min(tk, logits.size(-1))
|
||||
else:
|
||||
max_k = min(max_k, logits.size(-1))
|
||||
values, _ = torch.topk(logits, max_k, dim=-1)
|
||||
per_row_k = tk.clamp(max=max_k)
|
||||
thresholds = torch.full_like(logits[..., -1:], -float("inf"))
|
||||
positive = per_row_k > 0
|
||||
if positive.any():
|
||||
row_idx = torch.arange(logits.size(0), device=logits.device)[positive]
|
||||
thresholds[positive] = values[
|
||||
row_idx, per_row_k[positive] - 1
|
||||
].unsqueeze(-1)
|
||||
logits[logits < thresholds] = filter_value
|
||||
return logits
|
||||
thresholds = torch.topk(logits, k, dim=-1)[0][..., -1:]
|
||||
logits[logits < thresholds] = filter_value
|
||||
if tk > 0:
|
||||
k = min(tk, logits.size(-1))
|
||||
thresholds = torch.topk(logits, k, dim=-1)[0][..., -1:]
|
||||
logits[logits < thresholds] = filter_value
|
||||
return logits
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user