fix: report failed rollout requests

- Return structured finish and error reasons for synchronous generation
- Reject failed online rollout batches instead of training on empty responses
- Verify allocation and extension failures release metrics and KV state
This commit is contained in:
0z5a
2026-09-02 12:52:50 +08:00
parent c36846c8a4
commit 90de5bc1bd
6 changed files with 185 additions and 24 deletions
+12 -1
View File
@@ -2,8 +2,9 @@ import threading
import time
import uuid
from collections import deque
from dataclasses import dataclass
from enum import Enum
from typing import TYPE_CHECKING, Any, Callable, Deque, Dict, List, Optional
from typing import TYPE_CHECKING, Any, Callable, Deque, Dict, List, Literal, Optional
from tokenizers.decoders import DecodeStream
@@ -16,6 +17,16 @@ if TYPE_CHECKING:
STOP = object()
@dataclass(frozen=True)
class GenerationResult:
"""Structured terminal result for one synchronous generation request."""
token_ids: List[int]
logprobs: List[float]
finish_reason: Literal["stop", "length", "cancelled", "rejected"]
error_reason: Optional[str] = None
class StreamDecoder:
"""Incremental decoder backed by the tokenizers library's DecodeStream.