fix : handle zero-token batch generation
- return empty results without running inference for non-positive limits - keep scheduler batch outputs aligned with requested max_tokens - add engine and scheduler regression coverage
This commit is contained in:
@@ -288,6 +288,9 @@ class InferenceScheduler:
|
||||
t_max = seq_cap - len(ids)
|
||||
else:
|
||||
t_max = min(t_max, seq_cap - len(ids))
|
||||
if t_max <= 0:
|
||||
tasks.append(None)
|
||||
continue
|
||||
task = Task(
|
||||
task_id=f"batch_{uuid.uuid4().hex[:8]}",
|
||||
prompt_ids=list(ids),
|
||||
|
||||
@@ -146,6 +146,12 @@ class InferenceEngine:
|
||||
is_batch = isinstance(prompt, list)
|
||||
prompts = prompt if is_batch else [prompt]
|
||||
|
||||
if max_tokens is not None and max_tokens <= 0:
|
||||
if stream:
|
||||
return iter(())
|
||||
results = [""] * len(prompts)
|
||||
return results if is_batch else results[0]
|
||||
|
||||
if stream:
|
||||
return self._generate_streaming(
|
||||
prompts,
|
||||
|
||||
Reference in New Issue
Block a user