feat : add radix prefix cache

- replace hash-only lookup with page-granular radix matching
- keep partial pages private and cache only materialized KV prefixes
- integrate completed-request caching and add radix behavior tests
This commit is contained in:
2026-08-06 11:45:52 +08:00
parent 654e6eb0d1
commit b2230fefd8
5 changed files with 142 additions and 34 deletions
+7
View File
@@ -164,6 +164,13 @@ class InferenceScheduler:
while not self._stop_event.is_set():
finished = self._task_mgr.remove_finished_tasks(stop_ids)
for task in finished:
if task.status == TaskStatus.FINISHED:
cache.task_record_hashes(
task.task_id,
cache.task_cacheable_ids(
task.task_id, task.prompt_ids, task.output_ids
),
)
cache.task_free(task.task_id)
active = self._task_mgr.get_active_tasks()