refactor: use single-index access and update docs for cache architecture

- Replace all buffer[layer_id][loc] double indexing with buffer[layer_id, loc] single advanced indexing in cache.py and attention.py
- Revert KVStorage buffers back to 4D [n_layers, size, n_kv_heads, head_dim], remove leftover 3D reshape/view in MLA path
- Update docs/guides/inference.md, docs/developer/internals.md, docs/developer/architecture.md to reflect new PagePool/KVStorage/ReqToTokenPool/KVCache classes
This commit is contained in:
2026-07-30 17:47:04 +08:00
parent deb2d7e127
commit df979b4469
6 changed files with 88 additions and 116 deletions
+4 -4
View File
@@ -155,10 +155,10 @@ class ReqToTokenPool:
class KVStorage:
"""Token-level flat KV cache storage with NHD layout.
"""Token-level KV cache storage.
Buffers: [n_layers, size, n_kv_heads, head_dim]. Each token occupies
one contiguous row. Logical ordering is determined by ReqToTokenPool.
one slot indexed by ReqToTokenPool.
"""
def __init__(
@@ -185,8 +185,8 @@ class KVStorage:
return self.v_buffer[layer_id]
def set_kv_buffer(self, layer_id: int, loc: Tensor, k: Tensor, v: Tensor) -> None:
self.k_buffer[layer_id][loc] = k
self.v_buffer[layer_id][loc] = v
self.k_buffer[layer_id, loc] = k
self.v_buffer[layer_id, loc] = v
@dataclass