fix: report failed rollout requests
- Return structured finish and error reasons for synchronous generation - Reject failed online rollout batches instead of training on empty responses - Verify allocation and extension failures release metrics and KV state
This commit is contained in:
@@ -2,8 +2,9 @@ import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING, Any, Callable, Deque, Dict, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, Callable, Deque, Dict, List, Literal, Optional
|
||||
|
||||
from tokenizers.decoders import DecodeStream
|
||||
|
||||
@@ -16,6 +17,16 @@ if TYPE_CHECKING:
|
||||
STOP = object()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GenerationResult:
|
||||
"""Structured terminal result for one synchronous generation request."""
|
||||
|
||||
token_ids: List[int]
|
||||
logprobs: List[float]
|
||||
finish_reason: Literal["stop", "length", "cancelled", "rejected"]
|
||||
error_reason: Optional[str] = None
|
||||
|
||||
|
||||
class StreamDecoder:
|
||||
"""Incremental decoder backed by the tokenizers library's DecodeStream.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user