refactor: 页状态移入 PagedCache,Task 纯化为域对象
- PagedCache 增 task_alloc/task_free/task_extend/task_cached/task_record_hashes/make_table_tensor - Task 移除 page_table/n_pages/_prefix_cached_tokens/_pages_freed - Executor 移除 _PageState,页操作全部委托 PagedCache - CacheView.gather 截断逻辑下沉到 PagedCache.gather - 各类补充单行职责 docstring
This commit is contained in:
@@ -14,6 +14,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InferenceScheduler:
|
||||
"""Four-phase continuous batching loop: cleanup -> refill -> prefill -> decode."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: AutoModel,
|
||||
@@ -112,7 +114,7 @@ class InferenceScheduler:
|
||||
|
||||
groups: Dict[Tuple[int, int], List[Task]] = {}
|
||||
for t in to_prefill:
|
||||
key = (len(t.prompt_ids), t._prefix_cached_tokens)
|
||||
key = (len(t.prompt_ids), self._executor.get_cached_tokens(t))
|
||||
groups.setdefault(key, []).append(t)
|
||||
|
||||
for (prompt_len, start_pos), group in groups.items():
|
||||
|
||||
Reference in New Issue
Block a user