fix : handle zero-token batch generation

- return empty results without running inference for non-positive limits
- keep scheduler batch outputs aligned with requested max_tokens
- add engine and scheduler regression coverage
This commit is contained in:
2026-08-06 12:31:09 +08:00
parent 6f09b1d2ee
commit 5c180cfa90
4 changed files with 44 additions and 7 deletions
+3
View File
@@ -288,6 +288,9 @@ class InferenceScheduler:
t_max = seq_cap - len(ids)
else:
t_max = min(t_max, seq_cap - len(ids))
if t_max <= 0:
tasks.append(None)
continue
task = Task(
task_id=f"batch_{uuid.uuid4().hex[:8]}",
prompt_ids=list(ids),
+6
View File
@@ -146,6 +146,12 @@ class InferenceEngine:
is_batch = isinstance(prompt, list)
prompts = prompt if is_batch else [prompt]
if max_tokens is not None and max_tokens <= 0:
if stream:
return iter(())
results = [""] * len(prompts)
return results if is_batch else results[0]
if stream:
return self._generate_streaming(
prompts,