fix : handle zero-token batch generation
- return empty results without running inference for non-positive limits - keep scheduler batch outputs aligned with requested max_tokens - add engine and scheduler regression coverage
This commit is contained in:
@@ -288,6 +288,9 @@ class InferenceScheduler:
|
||||
t_max = seq_cap - len(ids)
|
||||
else:
|
||||
t_max = min(t_max, seq_cap - len(ids))
|
||||
if t_max <= 0:
|
||||
tasks.append(None)
|
||||
continue
|
||||
task = Task(
|
||||
task_id=f"batch_{uuid.uuid4().hex[:8]}",
|
||||
prompt_ids=list(ids),
|
||||
|
||||
Reference in New Issue
Block a user