From e08b3be893bebffc27a7e2a5072f527ca42f7c0f Mon Sep 17 00:00:00 2001 From: kunkunblueberry Date: Thu, 19 Nov 2026 23:10:47 +0800 Subject: [PATCH 1/5] feat(memory): size KV capacity from the post-warmup measurement Turn the RFC #192 step-1 preparation report into a KV block count, and let KVCacheManager grow its published range without reallocating tensors. - measured_kv_capacity() reads the final drained phase of the report: the driver's remaining memory is the physical budget, so weights, execution buffers, captured graphs and any live mapping are never subtracted twice. The utilization cap still bounds the instance, and the transient allowance and safety reserve are explicit inputs that are recorded with the result. Coverage gaps reported by the preparation (long contexts, random speculation, unreported batch/history combinations) travel with the plan, so an estimated margin is never presented as a measured one. - KVCacheManager accepts a capacity source whose tensors span the full reservation and whose physical backing grows by commit(). Requests are only ever handed published blocks, publishing requires the storage to report the blocks as backed, and published <= committed <= max is enforced, so no allocation can address unmapped memory. Release stays reference-based, as agreed for the storage contract: the owner drops references, the manager never unmaps. - With no capacity source nothing changes: tensors come from torch.empty and max == committed == published. Engine wiring (prepare_runtime -> plan -> commit -> publish -> replay and representative execution -> ready) needs #198 for the report and #194 for the storage, so it is the next step rather than part of this change. Signed-off-by: kunkunblueberry --- tests/test_measured_capacity.py | 243 ++++++++++++++++++++++++++++++ vllm_rlt/core/kv_cache_manager.py | 148 +++++++++++++++--- vllm_rlt/core/memory.py | 77 ++++++++++ 3 files changed, 450 insertions(+), 18 deletions(-) create mode 100644 tests/test_measured_capacity.py diff --git a/tests/test_measured_capacity.py b/tests/test_measured_capacity.py new file mode 100644 index 00000000..626da824 --- /dev/null +++ b/tests/test_measured_capacity.py @@ -0,0 +1,243 @@ +"""Measured KV capacity (RFC #192 step 2 handoff) and the publish gate. + +``measured_kv_capacity`` consumes the step-1 preparation report; the expected +values below are hand-computed from the KV layout and the stated margins, not +from the implementation. +""" + +import pytest +import torch + +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.core.memory import measured_kv_capacity + +MIB = 1 << 20 +GIB = 1024 * MIB +LAYERS, KV_HEADS, HEAD_DIM, BLOCK = 2, 1, 4, 2 + + +def report(*, free, total, gaps=()): + """One drained, post-cleanup sample as ``prepare_runtime`` reports it.""" + return dict( + phases=[ + dict( + name="prepared", + driver_free_bytes=free, + device_total_bytes=total, + allocated_bytes=0, + reserved_bytes=0, + ) + ], + coverage_gaps=tuple(gaps), + ) + + +def plan(**overrides): + parameters = dict( + report=report(free=60 * GIB, total=80 * GIB), + bytes_per_block=3 * MIB, + useful_blocks=10**6, + gpu_memory_utilization=0.9, + transient_bytes=4 * GIB, + safety_bytes=256 * MIB, + ) + parameters.update(overrides) + return measured_kv_capacity(**parameters) + + +def test_blocks_follow_the_measured_budget_without_double_counting(): + blocks, record = plan() + # resident = 80 - 60 = 20 GiB; cap = 72 GiB; remaining = min(60, 72 - 20) = 52 GiB + # budget = 52 GiB - 4 GiB - 256 MiB = 48896 MiB; 48896 // 3 = 16298 blocks + assert (record["resident_bytes"], record["utilization_cap_bytes"]) == (20 * GIB, int(72 * GIB)) + assert record["remaining_instance_bytes"] == 52 * GIB + assert record["kv_budget_bytes"] == 53248 * MIB - 4096 * MIB - 256 * MIB + assert blocks == 16298 + assert record["source"] == "measured_warmup" + assert blocks * record["bytes_per_block"] <= record["kv_budget_bytes"] + + +def test_driver_free_memory_is_bounded_by_the_utilization_cap(): + # Only 1 GiB is resident, so the 0.5 cap (40 GiB) binds before free memory. + blocks, record = plan( + report=report(free=79 * GIB, total=80 * GIB), + gpu_memory_utilization=0.5, + bytes_per_block=MIB, + transient_bytes=0, + safety_bytes=0, + ) + assert record["remaining_instance_bytes"] == 39 * GIB + assert blocks == 39 * 1024 + + +def test_each_margin_reduces_capacity_exactly(): + free = 20 * GIB + base, _ = plan( + report=report(free=free, total=80 * GIB), + transient_bytes=0, + safety_bytes=0, + gpu_memory_utilization=1.0, + ) + transient, _ = plan( + report=report(free=free, total=80 * GIB), + transient_bytes=1024 * 3 * MIB, + safety_bytes=0, + gpu_memory_utilization=1.0, + ) + safety, _ = plan( + report=report(free=free, total=80 * GIB), + transient_bytes=0, + safety_bytes=1024 * 3 * MIB, + gpu_memory_utilization=1.0, + ) + assert base - transient == 1024 + assert base - safety == 1024 + + +def test_useful_blocks_cap_the_measured_budget(): + blocks, record = plan(useful_blocks=100) + assert blocks == 100 + assert record["max_useful_blocks"] == 100 + + +def test_coverage_gaps_travel_with_the_plan(): + gaps = ("contexts beyond the reported workloads retain the estimated reserve",) + _, record = plan(report=report(free=60 * GIB, total=80 * GIB, gaps=gaps)) + assert record["coverage_gaps"] == gaps + + +def test_a_budget_below_one_block_is_rejected_with_the_margins(): + with pytest.raises(ValueError, match="remaining=.*transient=.*safety=.*block="): + plan( + report=report(free=8 * MIB, total=80 * GIB), + transient_bytes=0, + safety_bytes=0, + ) + + +@pytest.mark.parametrize( + "bad_report", + [None, {}, dict(phases=[]), dict(phases=[dict(driver_free_bytes=1)])], +) +def test_a_missing_or_incomplete_report_is_rejected(bad_report): + with pytest.raises(ValueError, match="report|phase"): + plan(report=bad_report) + + +@pytest.mark.parametrize( + "overrides, message", + [ + (dict(bytes_per_block=0), "bytes_per_block"), + (dict(useful_blocks=0), "useful_blocks"), + (dict(transient_bytes=-1), "transient_bytes"), + (dict(safety_bytes=-1), "safety_bytes"), + (dict(gpu_memory_utilization=0.0), "gpu_memory_utilization"), + ], +) +def test_invalid_inputs_are_rejected(overrides, message): + with pytest.raises(ValueError, match=message): + plan(**overrides) + + +class FakeStorage: + """Mapped storage stand-in: only ``committed_blocks`` has backing.""" + + def __init__(self, *, max_blocks, committed=0, dtype=torch.float32): + element = torch.empty((), dtype=dtype).element_size() + self.bytes_per_block = 2 * LAYERS * BLOCK * KV_HEADS * HEAD_DIM * element + shape = (max_blocks, LAYERS, BLOCK, KV_HEADS, HEAD_DIM) + self.tensors = ( + torch.zeros(shape, dtype=dtype), + torch.zeros(shape, dtype=dtype), + ) + self.max_blocks = max_blocks + self.dtype = dtype + self.committed_blocks = 0 + self.committed_bytes = 0 + self.commit(committed) + + def commit(self, blocks): + self.committed_blocks = max(self.committed_blocks, blocks) + self.committed_bytes = self.committed_blocks * self.bytes_per_block + return self.committed_blocks + + +def make_cache(storage=None, num_blocks=4, **kwargs): + parameters = dict( + num_layers=LAYERS, + num_kv_heads=KV_HEADS, + head_dim=HEAD_DIM, + num_blocks=num_blocks, + block_size=BLOCK, + max_loops=1, + ) + parameters.update(kwargs) + return KVCacheManager(**parameters, storage=storage) + + +def test_storage_backed_cache_starts_at_its_committed_capacity(): + storage = FakeStorage(max_blocks=10, committed=4) + cache = make_cache(storage, num_blocks=4) + assert (cache.published_blocks, cache.committed_blocks, cache.max_blocks) == (4, 4, 10) + assert cache.num_blocks == 4 + assert cache.num_free_blocks == 4 + assert cache.key_cache.shape == storage.tensors[0].shape + + +def test_publish_requires_committed_memory_and_grows_the_free_pool(): + storage = FakeStorage(max_blocks=10, committed=2) + cache = make_cache(storage, num_blocks=2) + assert cache.allocate("a", 4) # two published blocks + assert cache.allocate("b", 2) is False + with pytest.raises(ValueError, match="committed 2"): + cache.publish_capacity(8) + assert (cache.published_blocks, cache.num_free_blocks) == (2, 0) + + storage.commit(10) + assert cache.publish_capacity(10) == 10 + assert cache.num_free_blocks == 8 + assert cache.allocate("b", 2) + blocks = cache.get_block_table("b", 0) + assert set(blocks) & set(range(2, 10)), "growth must hand out the newly backed blocks" + key = torch.full((1, KV_HEADS, HEAD_DIM), 3.0) + cache.write(0, ["b"], [0], [0], key, key + 1) + assert torch.equal(cache.key_cache[blocks[0], 0, 0], key[0]) + + +def test_publish_is_monotonic_and_bounded(): + storage = FakeStorage(max_blocks=10, committed=10) + cache = make_cache(storage, num_blocks=2) + assert cache.publish_capacity(6) == 6 + with pytest.raises(ValueError, match="only grow"): + cache.publish_capacity(4) + with pytest.raises(ValueError, match="reservation holds"): + cache.publish_capacity(11) + assert cache.publish_capacity(6) == 6 # idempotent + assert cache.num_blocks == 6 + + +def test_publish_requires_storage(): + cache = make_cache(None) + with pytest.raises(RuntimeError, match="mapped storage"): + cache.publish_capacity(8) + + +def test_plain_cache_keeps_fixed_capacity(): + cache = make_cache(None, num_blocks=6) + assert cache.storage is None + assert (cache.max_blocks, cache.committed_blocks, cache.published_blocks) == (6, 6, 6) + + +def test_storage_must_be_committed_before_the_cache_is_built(): + storage = FakeStorage(max_blocks=10, committed=2) + with pytest.raises(ValueError, match="commit the pool"): + make_cache(storage, num_blocks=4) + + +def test_storage_must_cover_the_requested_blocks_and_layout(): + coarse = FakeStorage(max_blocks=2, committed=2) + with pytest.raises(ValueError, match="reserves"): + make_cache(coarse, num_blocks=4) + wrong_dtype = FakeStorage(max_blocks=10, committed=10, dtype=torch.bfloat16) + with pytest.raises(ValueError, match="do not match the KV layout"): + make_cache(wrong_dtype, num_blocks=4) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index c6187d77..4ff29f1f 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -18,6 +18,7 @@ from copy import deepcopy from dataclasses import dataclass, field from numbers import Integral +from typing import Protocol import torch @@ -166,6 +167,23 @@ class KVSnapshot: fp8_scale: float | None = None +class KVCapacitySource(Protocol): + """Storage that maps physical memory behind one fixed address range. + + ``tensors`` holds the K and V planes in that order and spans ``max_blocks``; + only the first ``committed_blocks`` are backed. ``commit(blocks)`` maps more + and returns how many blocks are fully backed. Release is by reference, as + with ``torch.empty``: the storage owner decides, not this manager. + """ + + max_blocks: int + committed_blocks: int + committed_bytes: int + tensors: tuple[torch.Tensor, ...] + + def commit(self, blocks: int) -> int: ... + + class KVCacheManager: """Own a fixed physical page pool shared by request/depth allocations. @@ -194,6 +212,7 @@ def __init__( attention: AttentionBackend | None = None, kv_cache_dtype: str = "auto", fp8_scale: float | None = None, + storage: KVCapacitySource | None = None, ): for name, value in ( ("num_layers", num_layers), @@ -252,6 +271,10 @@ def __init__( ) self.attention_info = self.attention.info self.backend = self.attention_info["selected_backend"] + self._storage = storage + self._watermark_ratio = watermark_ratio + if storage is not None and kv_cache_dtype != "auto": + raise ValueError("mapped KV storage does not carry FP8 scales yet") capability = "fp8_kv" if fp8_scale is None else "fp8_kv_static" if kv_cache_dtype != "auto" and not getattr(self.attention.capabilities, capability): raise ValueError( @@ -259,23 +282,35 @@ def __init__( f"with {'per-vector' if fp8_scale is None else 'fixed'} scales" ) shape = (num_blocks, num_layers, block_size, num_kv_heads, head_dim) - storage = torch.float8_e4m3fn if kv_cache_dtype == "fp8_e4m3" else dtype - self.key_cache = torch.empty(shape, device=self.device, dtype=storage) - self.value_cache = torch.empty_like(self.key_cache) - # One scale per stored vector; None for an unquantized pool. - self.key_scale = self.value_scale = None - if kv_cache_dtype == "fp8_e4m3": - # Imported fixed-scale pages use the same local scale as locally written pages. - scale = 1.0 if fp8_scale is None else fp8_scale - self.key_scale = torch.full(shape[:-1], scale, device=self.device, dtype=torch.float32) - self.value_scale = torch.full_like(self.key_scale, scale) - # Keep the existing per-vector storage/lifecycle for this prototype. FA - # borrows the immutable scalar; Torch/Triton continue reading the scale pool. - self._fixed_scale = ( - torch.tensor(fp8_scale, device=self.device, dtype=torch.float32) - if fp8_scale is not None - else None - ) + if storage is None: + storage_dtype = torch.float8_e4m3fn if kv_cache_dtype == "fp8_e4m3" else dtype + self.key_cache = torch.empty(shape, device=self.device, dtype=storage_dtype) + self.value_cache = torch.empty_like(self.key_cache) + # One scale per stored vector; None for an unquantized pool. + self.key_scale = self.value_scale = None + if kv_cache_dtype == "fp8_e4m3": + # Imported fixed-scale pages use the same local scale as locally written pages. + scale = 1.0 if fp8_scale is None else fp8_scale + self.key_scale = torch.full( + shape[:-1], scale, device=self.device, dtype=torch.float32 + ) + self.value_scale = torch.full_like(self.key_scale, scale) + # Keep the existing per-vector storage/lifecycle for this prototype. FA + # borrows the immutable scalar; Torch/Triton continue reading the scale pool. + self._fixed_scale = ( + torch.tensor(fp8_scale, device=self.device, dtype=torch.float32) + if fp8_scale is not None + else None + ) + self.max_blocks = num_blocks + self.committed_blocks = num_blocks + else: + self.key_cache, self.value_cache = self._attach_storage(storage, num_blocks) + self.key_scale = self.value_scale = None + self._fixed_scale = None + # Requests only ever receive published blocks, and publishing requires + # committed physical memory, so published <= committed <= max holds. + self.published_blocks = num_blocks # Resolve an implicit CUDA index to the actual storage device once. self.device = self.key_cache.device self._free_blocks = list(reversed(range(num_blocks))) @@ -291,7 +326,7 @@ def __init__( self.enable_prefix_caching = enable_prefix_caching self.incremental_allocation = incremental_allocation # Full allocation already reserves each request's maximum KV capacity. - self.watermark_blocks = int(watermark_ratio * num_blocks) if incremental_allocation else 0 + self.watermark_blocks = self._watermark(num_blocks) self._refs = [0] * num_blocks self._prefixes = OrderedDict() self._pending_prefixes = [] @@ -324,6 +359,83 @@ def bytes_per_block(self) -> int: self.kv_cache_dtype, ) + @property + def storage(self) -> KVCapacitySource | None: + return self._storage + + def _watermark(self, blocks: int) -> int: + return int(self._watermark_ratio * blocks) if self.incremental_allocation else 0 + + def _attach_storage(self, storage: KVCapacitySource, num_blocks: int): + """Take the K/V planes from a pre-reserved pool instead of allocating them.""" + expected = kv_bytes_per_block( + self.num_layers, + self.block_size, + self.num_kv_heads, + self.head_dim, + torch.empty((), dtype=self.dtype).element_size(), + self.kv_cache_dtype, + ) + if storage.max_blocks < num_blocks: + raise ValueError( + f"storage reserves {storage.max_blocks} blocks but {num_blocks} were requested" + ) + if storage.committed_blocks < num_blocks: + raise ValueError( + f"storage committed {storage.committed_blocks} blocks but {num_blocks} " + "were requested; commit the pool before building the cache" + ) + if storage.committed_bytes < num_blocks * expected: + raise ValueError( + f"storage reports {storage.committed_bytes} committed bytes but " + f"{num_blocks} blocks need {num_blocks * expected}" + ) + keys, values = storage.tensors + shape = ( + storage.max_blocks, + self.num_layers, + self.block_size, + self.num_kv_heads, + self.head_dim, + ) + for name, tensor in (("keys", keys), ("values", values)): + if tensor.dtype != self.dtype or tuple(tensor.shape) != shape: + raise ValueError(f"storage {name} do not match the KV layout") + self.max_blocks = storage.max_blocks + self.committed_blocks = storage.committed_blocks + return keys, values + + def publish_capacity(self, blocks: int) -> int: + """Grow the allocatable pool after the storage mapped more blocks. + + The K/V tensors keep their address, shape and stride; only the free pool + and the published count grow. Uncommitted blocks are never published, so + no allocation can reference unmapped memory. + """ + if self._storage is None: + raise RuntimeError("capacity publishing requires mapped storage") + if type(blocks) is not int or blocks < self.published_blocks: + raise ValueError("published capacity can only grow") + if blocks > self.max_blocks: + raise ValueError( + f"cannot publish {blocks} blocks; the reservation holds {self.max_blocks}" + ) + # Re-read the pool: a commit may have happened since the last publish. + self.committed_blocks = self._storage.committed_blocks + if blocks > self.committed_blocks: + raise ValueError( + f"cannot publish {blocks} blocks; the storage committed {self.committed_blocks}" + ) + if blocks == self.published_blocks: + return blocks + self._refs.extend([0] * (blocks - self.published_blocks)) + # _claim pops from the tail, so later blocks follow in ascending order. + self._free_blocks.extend(reversed(range(self.published_blocks, blocks))) + self.published_blocks = blocks + self.num_blocks = blocks + self.watermark_blocks = self._watermark(blocks) + return blocks + def required_blocks(self, max_tokens: int) -> int: if not isinstance(max_tokens, Integral) or isinstance(max_tokens, bool) or max_tokens <= 0: raise ValueError("max_tokens must be a positive integer") diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index 1c52c16c..50c373fe 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -19,6 +19,83 @@ def budget_blocks(available_bytes, bytes_per_block): return blocks +def measurement_budget(report): + """Driver free/total bytes and coverage gaps from a startup preparation report. + + ``report`` is step 1 of RFC #192 (``StageEngine.prepare_runtime``): boundary + samples taken after draining, with the allocator cache retained. Only the + last sample sizes capacity, because it is the state after warmup and capture. + """ + if not report: + raise ValueError("a measured KV plan requires a startup preparation report") + phases = report.get("phases") or () + if not phases: + raise ValueError("the startup report contains no phase samples") + final = phases[-1] + free, total = final.get("driver_free_bytes"), final.get("device_total_bytes") + if type(free) is not int or type(total) is not int or total <= 0 or free < 0: + raise ValueError("the final startup phase lacks valid driver free/total bytes") + return free, total, tuple(report.get("coverage_gaps") or ()) + + +def measured_kv_capacity( + *, + report, + bytes_per_block, + useful_blocks, + gpu_memory_utilization, + transient_bytes, + safety_bytes, +): + """Blocks to commit after warmup and capture, with every margin recorded. + + The physical budget is the driver's remaining memory, which already excludes + weights, execution buffers, captured graphs and any live mapping, so none of + them is subtracted a second time. ``transient_bytes`` covers execution above + that persistent baseline and ``safety_bytes`` the fixed reserve. + """ + for name, value in ( + ("bytes_per_block", bytes_per_block), + ("useful_blocks", useful_blocks), + ): + if type(value) is not int or value < 1: + raise ValueError(f"{name} must be a positive integer") + for name, value in ( + ("transient_bytes", transient_bytes), + ("safety_bytes", safety_bytes), + ): + if type(value) is not int or value < 0: + raise ValueError(f"{name} must be a nonnegative integer") + if not 0 < gpu_memory_utilization <= 1: + raise ValueError("gpu_memory_utilization must be in (0, 1]") + free, total, gaps = measurement_budget(report) + resident = total - free + # What the utilization cap still allows, bounded by measured free memory. + remaining = min(free, int(total * gpu_memory_utilization) - resident) + budget = remaining - transient_bytes - safety_bytes + if budget < bytes_per_block: + raise ValueError( + "no KV capacity fits the measured budget: " + f"remaining={remaining} transient={transient_bytes} " + f"safety={safety_bytes} block={bytes_per_block}" + ) + blocks = min(budget // bytes_per_block, useful_blocks) + return blocks, dict( + source="measured_warmup", + bytes_per_block=bytes_per_block, + driver_free_bytes=free, + device_total_bytes=total, + resident_bytes=resident, + utilization_cap_bytes=int(total * gpu_memory_utilization), + remaining_instance_bytes=remaining, + transient_bytes=transient_bytes, + safety_bytes=safety_bytes, + kv_budget_bytes=budget, + max_useful_blocks=useful_blocks, + coverage_gaps=gaps, + ) + + def graph_memory_reserve(model, scheduler, execution, speculative=None): """Headroom per pool plus retained speculative coda logits across all entries. From f1b40c3b508fa90387308354b43f56fdbce488a1 Mon Sep 17 00:00:00 2001 From: kunkunblueberry Date: Fri, 20 Nov 2026 00:01:50 +0800 Subject: [PATCH 2/5] refactor(kv): separate torch and mapped plane setup Move the tensor/scale allocation and the mapped-pool binding out of KVCacheManager.__init__ into _open_torch_planes and _open_mapped_planes. The FP8 capability rules, the allocation and the per-vector scale lifecycle now sit in one method, and the mapped path states its own FP8 limitation instead of being special-cased at the top of __init__. The capacity fields keep the same meaning: both paths set max_blocks and committed_blocks, and __init__ publishes only afterwards. Behaviour is unchanged: the FP8 KV, backend-selection, Dory, prepared-KV and KV-cache suites pass as before, the full CPU suite keeps the same 32 pre-existing failures, and no line from the FP8 work merged in #129 was dropped (diffed the fp8-related lines against main; the only differences are the new mapped-path guard and the local `storage` -> `storage_dtype` rename that the new parameter required). Signed-off-by: kunkunblueberry --- vllm_rlt/core/kv_cache_manager.py | 96 ++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 4ff29f1f..36eb8103 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -273,41 +273,18 @@ def __init__( self.backend = self.attention_info["selected_backend"] self._storage = storage self._watermark_ratio = watermark_ratio - if storage is not None and kv_cache_dtype != "auto": - raise ValueError("mapped KV storage does not carry FP8 scales yet") - capability = "fp8_kv" if fp8_scale is None else "fp8_kv_static" - if kv_cache_dtype != "auto" and not getattr(self.attention.capabilities, capability): - raise ValueError( - f"the {self.attention_info['backend']} backend does not read FP8 KV " - f"with {'per-vector' if fp8_scale is None else 'fixed'} scales" - ) - shape = (num_blocks, num_layers, block_size, num_kv_heads, head_dim) if storage is None: - storage_dtype = torch.float8_e4m3fn if kv_cache_dtype == "fp8_e4m3" else dtype - self.key_cache = torch.empty(shape, device=self.device, dtype=storage_dtype) - self.value_cache = torch.empty_like(self.key_cache) - # One scale per stored vector; None for an unquantized pool. - self.key_scale = self.value_scale = None - if kv_cache_dtype == "fp8_e4m3": - # Imported fixed-scale pages use the same local scale as locally written pages. - scale = 1.0 if fp8_scale is None else fp8_scale - self.key_scale = torch.full( - shape[:-1], scale, device=self.device, dtype=torch.float32 - ) - self.value_scale = torch.full_like(self.key_scale, scale) - # Keep the existing per-vector storage/lifecycle for this prototype. FA - # borrows the immutable scalar; Torch/Triton continue reading the scale pool. - self._fixed_scale = ( - torch.tensor(fp8_scale, device=self.device, dtype=torch.float32) - if fp8_scale is not None - else None + self._open_torch_planes( + num_blocks=num_blocks, + block_size=block_size, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + dtype=dtype, + kv_cache_dtype=kv_cache_dtype, + fp8_scale=fp8_scale, ) - self.max_blocks = num_blocks - self.committed_blocks = num_blocks else: - self.key_cache, self.value_cache = self._attach_storage(storage, num_blocks) - self.key_scale = self.value_scale = None - self._fixed_scale = None + self._open_mapped_planes(storage, num_blocks) # Requests only ever receive published blocks, and publishing requires # committed physical memory, so published <= committed <= max holds. self.published_blocks = num_blocks @@ -366,8 +343,55 @@ def storage(self) -> KVCapacitySource | None: def _watermark(self, blocks: int) -> int: return int(self._watermark_ratio * blocks) if self.incremental_allocation else 0 - def _attach_storage(self, storage: KVCapacitySource, num_blocks: int): - """Take the K/V planes from a pre-reserved pool instead of allocating them.""" + def _open_torch_planes( + self, + *, + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_dim: int, + dtype: torch.dtype, + kv_cache_dtype: str, + fp8_scale: float | None, + ) -> None: + """Allocate the K/V planes with the PyTorch allocator, including FP8 scales.""" + capability = "fp8_kv" if fp8_scale is None else "fp8_kv_static" + if kv_cache_dtype != "auto" and not getattr(self.attention.capabilities, capability): + raise ValueError( + f"the {self.attention_info['backend']} backend does not read FP8 KV " + f"with {'per-vector' if fp8_scale is None else 'fixed'} scales" + ) + shape = (num_blocks, self.num_layers, block_size, num_kv_heads, head_dim) + storage_dtype = torch.float8_e4m3fn if kv_cache_dtype == "fp8_e4m3" else dtype + self.key_cache = torch.empty(shape, device=self.device, dtype=storage_dtype) + self.value_cache = torch.empty_like(self.key_cache) + # One scale per stored vector; None for an unquantized pool. + self.key_scale = self.value_scale = None + if kv_cache_dtype == "fp8_e4m3": + # Imported fixed-scale pages use the same local scale as locally written pages. + scale = 1.0 if fp8_scale is None else fp8_scale + self.key_scale = torch.full(shape[:-1], scale, device=self.device, dtype=torch.float32) + self.value_scale = torch.full_like(self.key_scale, scale) + # Keep the existing per-vector storage/lifecycle for this prototype. FA + # borrows the immutable scalar; Torch/Triton continue reading the scale pool. + self._fixed_scale = ( + torch.tensor(fp8_scale, device=self.device, dtype=torch.float32) + if fp8_scale is not None + else None + ) + self.max_blocks = num_blocks + self.committed_blocks = num_blocks + + def _open_mapped_planes(self, storage: KVCapacitySource, num_blocks: int) -> None: + """Bind the K/V planes to a pool that owns their physical mappings. + + FP8 KV needs per-vector scales, which the pool does not carry yet, so it + is rejected here rather than writing unquantized bytes into a quantized + cache. Only the pool's backed blocks are exposed; publishing them is the + manager's decision, not the storage's. + """ + if self.kv_cache_dtype != "auto": + raise ValueError("mapped KV storage does not carry FP8 scales yet") expected = kv_bytes_per_block( self.num_layers, self.block_size, @@ -401,9 +425,11 @@ def _attach_storage(self, storage: KVCapacitySource, num_blocks: int): for name, tensor in (("keys", keys), ("values", values)): if tensor.dtype != self.dtype or tuple(tensor.shape) != shape: raise ValueError(f"storage {name} do not match the KV layout") + self.key_cache, self.value_cache = keys, values + self.key_scale = self.value_scale = None + self._fixed_scale = None self.max_blocks = storage.max_blocks self.committed_blocks = storage.committed_blocks - return keys, values def publish_capacity(self, blocks: int) -> int: """Grow the allocatable pool after the storage mapped more blocks. From 9d93ef9c59ad1126349fec1b858c043a9515e630 Mon Sep 17 00:00:00 2001 From: kunkunblueberry Date: Fri, 20 Nov 2026 00:24:07 +0800 Subject: [PATCH 3/5] fix(memory): return total capacity and clamp it by the reservation measured_kv_capacity() reported the measured delta as if it were the total block count. Driver free memory excludes the bootstrap prefix that is already mapped, so an integration would have committed only the delta and under-counted the pool by the bootstrap size: with the Ouro-1.4B configuration, 22894 instead of 44746 blocks, about 64 GiB of KV capacity. - existing_blocks adds the already-mapped prefix to the measured delta and keeps it out of the free-memory sizing, so it is never counted twice. - reservation_blocks clamps the result to what the pool can actually grow to, and the truncation is recorded as a coverage gap. - limited_by reports which bound applied (budget, useful_blocks, reservation). - A bootstrap-only pool is now accepted when no additional capacity fits, instead of failing although the existing prefix is usable. - A zero transient allowance is rejected when the preparation reported coverage gaps, because an unexercised path cannot be sized as if it were measured. - KVCacheManager rejects a capacity source that claims more committed blocks than it reserves. Verified on the real checkpoint: existing 21852 blocks, additional 22894, total 44746, and only the additional part spends the 67.07 GiB budget. Signed-off-by: kunkunblueberry --- tests/test_measured_capacity.py | 57 +++++++++++++++++++++++++++++++ vllm_rlt/core/kv_cache_manager.py | 5 +++ vllm_rlt/core/memory.py | 46 ++++++++++++++++++++----- 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/tests/test_measured_capacity.py b/tests/test_measured_capacity.py index 626da824..42692953 100644 --- a/tests/test_measured_capacity.py +++ b/tests/test_measured_capacity.py @@ -241,3 +241,60 @@ def test_storage_must_cover_the_requested_blocks_and_layout(): wrong_dtype = FakeStorage(max_blocks=10, committed=10, dtype=torch.bfloat16) with pytest.raises(ValueError, match="do not match the KV layout"): make_cache(wrong_dtype, num_blocks=4) + + +def test_existing_bootstrap_blocks_are_counted_once_in_the_total(): + # The bootstrap prefix is already mapped, so it sits outside `free` and must + # extend the measured delta rather than being sized from free again. + capacity, record = plan(existing_blocks=100) + assert record["additional_blocks"] == 16298 + assert capacity == 16398 + assert record["existing_blocks"] == 100 + # Only the new part spends budget. + assert (capacity - 100) * record["bytes_per_block"] <= record["kv_budget_bytes"] + + +def test_bootstrap_only_capacity_is_usable_without_growth(): + capacity, record = plan( + report=report(free=8 * MIB, total=80 * GIB), + existing_blocks=8, + transient_bytes=0, + safety_bytes=0, + ) + assert capacity == 8 + assert record["additional_blocks"] == 0 + assert record["limited_by"] == "budget" + + +def test_the_reservation_truncates_the_measured_capacity(): + capacity, record = plan(reservation_blocks=1000) + assert capacity == 1000 + assert record["limited_by"] == "reservation" + assert any("exceeds the reservation" in gap for gap in record["coverage_gaps"]) + + +def test_the_useful_block_cap_binds_before_the_reservation(): + capacity, record = plan(useful_blocks=100, reservation_blocks=1000) + assert capacity == 100 + assert record["limited_by"] == "useful_blocks" + assert not any("exceeds the reservation" in gap for gap in record["coverage_gaps"]) + + +def test_coverage_gaps_reject_a_zero_transient_allowance(): + gaps = ("random block speculation retains the estimated transient reserve",) + with pytest.raises(ValueError, match="coverage gaps require an explicit transient"): + plan( + report=report(free=60 * GIB, total=80 * GIB, gaps=gaps), + transient_bytes=0, + safety_bytes=0, + ) + # The same report is accepted once an allowance is stated, and the gap travels with it. + _, record = plan(report=report(free=60 * GIB, total=80 * GIB, gaps=gaps)) + assert record["coverage_gaps"] == gaps + + +def test_storage_may_not_report_more_committed_blocks_than_it_reserves(): + storage = FakeStorage(max_blocks=4, committed=4) + storage.committed_blocks = 5 # a storage-side accounting bug + with pytest.raises(ValueError, match="reserves only"): + make_cache(storage, num_blocks=4) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 36eb8103..8146f8f8 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -392,6 +392,11 @@ def _open_mapped_planes(self, storage: KVCapacitySource, num_blocks: int) -> Non """ if self.kv_cache_dtype != "auto": raise ValueError("mapped KV storage does not carry FP8 scales yet") + if storage.committed_blocks > storage.max_blocks: + raise ValueError( + f"storage reports {storage.committed_blocks} committed blocks but " + f"reserves only {storage.max_blocks}" + ) expected = kv_bytes_per_block( self.num_layers, self.block_size, diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index 50c373fe..c9dff496 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -46,13 +46,20 @@ def measured_kv_capacity( gpu_memory_utilization, transient_bytes, safety_bytes, + existing_blocks=0, + reservation_blocks=None, ): - """Blocks to commit after warmup and capture, with every margin recorded. + """Total blocks the pool may hold after warmup and capture. - The physical budget is the driver's remaining memory, which already excludes - weights, execution buffers, captured graphs and any live mapping, so none of - them is subtracted a second time. ``transient_bytes`` covers execution above - that persistent baseline and ``safety_bytes`` the fixed reserve. + ``free`` memory already excludes weights, execution buffers, captured graphs + and anything already mapped, including ``existing_blocks`` (the bootstrap + prefix). The measured capacity is therefore additional, and the cumulative + ``commit(total)`` of a storage maps only the new part. + + ``transient_bytes`` covers execution above that persistent baseline and + ``safety_bytes`` the fixed reserve. The result is clamped by the logical + ``useful_blocks`` and, when the pool cannot grow that far, by + ``reservation_blocks``; the binding limit and every margin are recorded. """ for name, value in ( ("bytes_per_block", bytes_per_block), @@ -63,24 +70,43 @@ def measured_kv_capacity( for name, value in ( ("transient_bytes", transient_bytes), ("safety_bytes", safety_bytes), + ("existing_blocks", existing_blocks), ): if type(value) is not int or value < 0: raise ValueError(f"{name} must be a nonnegative integer") + if reservation_blocks is not None and ( + type(reservation_blocks) is not int or reservation_blocks < 1 + ): + raise ValueError("reservation_blocks must be a positive integer") if not 0 < gpu_memory_utilization <= 1: raise ValueError("gpu_memory_utilization must be in (0, 1]") free, total, gaps = measurement_budget(report) + if gaps and not transient_bytes: + # The preparation reported paths it did not exercise. Sizing them as if + # measured would publish a pool that cannot serve them. + raise ValueError(f"coverage gaps require an explicit transient allowance: {gaps[0]}") resident = total - free # What the utilization cap still allows, bounded by measured free memory. remaining = min(free, int(total * gpu_memory_utilization) - resident) budget = remaining - transient_bytes - safety_bytes - if budget < bytes_per_block: + additional = max(0, budget // bytes_per_block) + capacity = existing_blocks + additional + if capacity < 1: raise ValueError( "no KV capacity fits the measured budget: " f"remaining={remaining} transient={transient_bytes} " f"safety={safety_bytes} block={bytes_per_block}" ) - blocks = min(budget // bytes_per_block, useful_blocks) - return blocks, dict( + limit = "budget" + if capacity > useful_blocks: + capacity, limit = useful_blocks, "useful_blocks" + if reservation_blocks is not None and capacity > reservation_blocks: + capacity, limit = reservation_blocks, "reservation" + gaps = ( + *gaps, + "measured capacity exceeds the reservation; committed capacity truncated", + ) + return capacity, dict( source="measured_warmup", bytes_per_block=bytes_per_block, driver_free_bytes=free, @@ -91,7 +117,11 @@ def measured_kv_capacity( transient_bytes=transient_bytes, safety_bytes=safety_bytes, kv_budget_bytes=budget, + existing_blocks=existing_blocks, + additional_blocks=additional, max_useful_blocks=useful_blocks, + reservation_blocks=reservation_blocks, + limited_by=limit, coverage_gaps=gaps, ) From 32a4a792dff87e0f78af411f4e67899e5b513f6a Mon Sep 17 00:00:00 2001 From: kunkunblueberry Date: Sat, 21 Nov 2026 16:29:53 +0800 Subject: [PATCH 4/5] feat(memory): derive the transient allowance and guard the measured commit Two pieces the agreed startup ordering needs before the engine can call it. - transient_allowance() derives the margin from each phase's own peak minus its end state, keeps the larger of that and the configured estimate, and scales it by an overlap factor for supported concurrency. An unexercised path keeps the estimate instead of being sized as though it were measured. - commit_measured_capacity() enforces the preconditions in one place: the report must describe the pool the storage already backs, the plan is clamped to the storage's reservation, and the storage has to report at least the planned blocks as backed. It returns that count, and the caller publishes exactly it. The second one is also what keeps the missing commit() bound in a storage implementation from turning into an overshoot: the plan cannot exceed the reservation, and a storage that under- or over-commits is rejected with the numbers in the message. Covered by tests with hand-computed expectations, including a storage that silently ignores the commit and one that reports beyond its reservation. Signed-off-by: kunkunblueberry --- tests/test_measured_capacity.py | 100 +++++++++++++++++++++++++++++++- vllm_rlt/core/memory.py | 86 +++++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 3 deletions(-) diff --git a/tests/test_measured_capacity.py b/tests/test_measured_capacity.py index 42692953..e24fe1cb 100644 --- a/tests/test_measured_capacity.py +++ b/tests/test_measured_capacity.py @@ -9,16 +9,20 @@ import torch from vllm_rlt.core.kv_cache_manager import KVCacheManager -from vllm_rlt.core.memory import measured_kv_capacity +from vllm_rlt.core.memory import ( + commit_measured_capacity, + measured_kv_capacity, + transient_allowance, +) MIB = 1 << 20 GIB = 1024 * MIB LAYERS, KV_HEADS, HEAD_DIM, BLOCK = 2, 1, 4, 2 -def report(*, free, total, gaps=()): +def report(*, free, total, gaps=(), blocks=None): """One drained, post-cleanup sample as ``prepare_runtime`` reports it.""" - return dict( + sample = dict( phases=[ dict( name="prepared", @@ -30,6 +34,9 @@ def report(*, free, total, gaps=()): ], coverage_gaps=tuple(gaps), ) + if blocks is not None: + sample["num_blocks"] = blocks + return sample def plan(**overrides): @@ -298,3 +305,90 @@ def test_storage_may_not_report_more_committed_blocks_than_it_reserves(): storage.committed_blocks = 5 # a storage-side accounting bug with pytest.raises(ValueError, match="reserves only"): make_cache(storage, num_blocks=4) + + +def test_transient_allowance_takes_the_larger_of_measured_and_estimate(): + phases = [ + dict(name="initialized", allocated_bytes=100 * MIB, peak_allocated_bytes=400 * MIB), + dict(name="prepared", allocated_bytes=300 * MIB, peak_allocated_bytes=500 * MIB), + ] + value, record = transient_allowance({"phases": phases}, estimate_bytes=64 * MIB) + assert record["measured_transient_bytes"] == 300 * MIB # max(300, 500 - 300) + assert value == 300 * MIB + assert record["sampled_phases"] == 2 + + value, _ = transient_allowance({"phases": phases}, estimate_bytes=512 * MIB) + assert value == 512 * MIB # an unexercised path keeps the estimate + + value, _ = transient_allowance({"phases": phases}, estimate_bytes=0, overlap_factor=2) + assert value == 600 * MIB # a serial peak does not bound a concurrent envelope + + +def test_transient_allowance_ignores_phases_without_peak_samples(): + value, record = transient_allowance({"phases": [dict(name="cpu")]}, estimate_bytes=MIB) + assert record["sampled_phases"] == 0 + assert value == MIB + + +@pytest.mark.parametrize( + "kwargs", [dict(estimate_bytes=-1), dict(estimate_bytes=0, overlap_factor=0)] +) +def test_transient_allowance_rejects_invalid_inputs(kwargs): + with pytest.raises(ValueError): + transient_allowance({"phases": []}, **kwargs) + + +def commit_args(**kwargs): + """Arguments for ``commit_measured_capacity`` on the 60/80 GiB sample.""" + options = dict( + report=report(free=60 * GIB, total=80 * GIB, blocks=1000), + bytes_per_block=3 * MIB, + useful_blocks=131072, + gpu_memory_utilization=0.9, + transient_bytes=4 * GIB, + safety_bytes=256 * MIB, + ) + options.update(kwargs) + return options + + +def test_commit_measured_capacity_maps_the_plan_and_returns_backed_blocks(): + storage = FakeStorage(max_blocks=131072, committed=1000) + backed, record = commit_measured_capacity(storage, **commit_args()) + assert backed == 1000 + 16298 + assert storage.committed_blocks == backed + assert record["committed_blocks"] == backed + assert record["storage"] == "mapped" + assert record["limited_by"] == "budget" + + +def test_commit_measured_capacity_requires_the_report_to_describe_this_pool(): + storage = FakeStorage(max_blocks=131072, committed=1000) + drifted = report(free=60 * GIB, total=80 * GIB, blocks=999) + with pytest.raises(ValueError, match="storage backs"): + commit_measured_capacity(storage, **commit_args(report=drifted)) + no_pool = report(free=60 * GIB, total=80 * GIB) + with pytest.raises(ValueError, match="num_blocks"): + commit_measured_capacity(storage, **commit_args(report=no_pool)) + + +def test_commit_measured_capacity_rejects_a_storage_that_under_commits(): + storage = FakeStorage(max_blocks=131072, committed=1000) + storage.commit = lambda blocks: storage.committed_blocks + with pytest.raises(RuntimeError, match="after committing"): + commit_measured_capacity(storage, **commit_args()) + + +def test_commit_measured_capacity_rejects_backing_beyond_the_reservation(): + storage = FakeStorage(max_blocks=131072, committed=1000) + storage.commit = lambda blocks: storage.max_blocks + 1 + with pytest.raises(RuntimeError, match="beyond its"): + commit_measured_capacity(storage, **commit_args()) + + +def test_commit_measured_capacity_clamps_to_the_reservation(): + storage = FakeStorage(max_blocks=5000, committed=1000) + backed, record = commit_measured_capacity(storage, **commit_args()) + assert record["limited_by"] == "reservation" + assert backed == 5000 + assert any("exceeds the reservation" in gap for gap in record["coverage_gaps"]) diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index c9dff496..f9d0bfec 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -126,6 +126,92 @@ def measured_kv_capacity( ) +def transient_allowance(report, *, estimate_bytes, overlap_factor=1): + """Allowance for execution above the persistent baseline of a phase. + + A phase records its end state and its own peak, so ``peak - allocated`` is + what that phase needed on top of what it kept. The largest such delta is + measured; ``estimate_bytes`` covers paths the preparation did not exercise + and must stay in force. ``overlap_factor`` covers supported concurrency, + because a serial peak understates a concurrent envelope. The larger of the + two is returned, so an unmeasured path is never sized as if measured. + """ + if type(estimate_bytes) is not int or estimate_bytes < 0: + raise ValueError("estimate_bytes must be a nonnegative integer") + if type(overlap_factor) is not int or overlap_factor < 1: + raise ValueError("overlap_factor must be an integer of at least 1") + phases = (report or {}).get("phases") or () + measured = 0 + sampled = 0 + for phase in phases: + peak, allocated = phase.get("peak_allocated_bytes"), phase.get("allocated_bytes") + if type(peak) is not int or type(allocated) is not int: + continue + sampled += 1 + measured = max(measured, peak - allocated) + value = max(measured * overlap_factor, estimate_bytes) + return value, dict( + transient_bytes=value, + measured_transient_bytes=measured, + estimate_bytes=estimate_bytes, + overlap_factor=overlap_factor, + sampled_phases=sampled, + ) + + +def commit_measured_capacity( + storage, + *, + report, + bytes_per_block, + useful_blocks, + gpu_memory_utilization, + transient_bytes, + safety_bytes, +): + """Plan the measured capacity, map it, and return the publishable block count. + + Enforces the preconditions of the agreed ordering: the report describes the + pool the storage already backs, the plan never exceeds the reservation, and + the storage must report at least the planned blocks as backed. The caller + publishes exactly the returned count and nothing else. + """ + existing = (report or {}).get("num_blocks") + if type(existing) is not int or existing < 0: + raise ValueError("the startup report must record the current pool as num_blocks") + if existing > storage.max_blocks: + raise ValueError( + f"the report describes {existing} blocks but the storage reserves {storage.max_blocks}" + ) + if storage.committed_blocks != existing: + # The pool changed after the sample, so the measured free memory no + # longer describes this storage. + raise ValueError( + f"the report describes {existing} blocks but the storage backs " + f"{storage.committed_blocks}" + ) + capacity, record = measured_kv_capacity( + report=report, + bytes_per_block=bytes_per_block, + useful_blocks=useful_blocks, + gpu_memory_utilization=gpu_memory_utilization, + transient_bytes=transient_bytes, + safety_bytes=safety_bytes, + existing_blocks=existing, + reservation_blocks=storage.max_blocks, + ) + backed = storage.commit(capacity) + if type(backed) is not int or backed < capacity: + raise RuntimeError(f"storage reported {backed} backed blocks after committing {capacity}") + if backed > storage.max_blocks: + raise RuntimeError( + f"storage reported {backed} backed blocks beyond its " + f"{storage.max_blocks}-block reservation" + ) + record.update(committed_blocks=backed, storage="mapped") + return backed, record + + def graph_memory_reserve(model, scheduler, execution, speculative=None): """Headroom per pool plus retained speculative coda logits across all entries. From ab3a946de3ae6057d969959983265f94e721d398 Mon Sep 17 00:00:00 2001 From: kunkunblueberry Date: Sat, 21 Nov 2026 17:45:07 +0800 Subject: [PATCH 5/5] feat(kv): accept FP8 over a mapped pool behind a scale hook The mapped-pool path rejected FP8 because a pool carried no per-vector scales. Make the pool declare the format it carries and bind FP8 through an explicit hook: a source that exposes scale_tensors (K first) supplies the per-vector scales, and a fixed fp8_scale needs no per-vector storage. A pool that declares neither is still rejected, but the error names the hook. The measured commit now also enforces the two caller policies the ordering depends on: the trial pool must already back at least one block, and a measured capacity that the reservation would truncate back to the pre-warmup estimate is an error instead of a silent clamp, since that clamp defeats the purpose of measuring. allow_reservation_truncation opts back in. Document that gpu_memory_utilization keeps capping a fraction of the whole device rather than of post-baseline free memory. --- tests/test_measured_capacity.py | 113 ++++++++++++++++++++++++++++-- vllm_rlt/core/kv_cache_manager.py | 58 ++++++++++++--- vllm_rlt/core/memory.py | 24 +++++++ 3 files changed, 179 insertions(+), 16 deletions(-) diff --git a/tests/test_measured_capacity.py b/tests/test_measured_capacity.py index e24fe1cb..3f4df296 100644 --- a/tests/test_measured_capacity.py +++ b/tests/test_measured_capacity.py @@ -5,6 +5,8 @@ from the implementation. """ +from types import SimpleNamespace + import pytest import torch @@ -149,14 +151,36 @@ def test_invalid_inputs_are_rejected(overrides, message): class FakeStorage: """Mapped storage stand-in: only ``committed_blocks`` has backing.""" - def __init__(self, *, max_blocks, committed=0, dtype=torch.float32): - element = torch.empty((), dtype=dtype).element_size() - self.bytes_per_block = 2 * LAYERS * BLOCK * KV_HEADS * HEAD_DIM * element + def __init__( + self, + *, + max_blocks, + committed=0, + dtype=torch.float32, + kv_cache_dtype="auto", + with_scales=False, + ): + self.kv_cache_dtype = kv_cache_dtype + if kv_cache_dtype == "auto": + vector = HEAD_DIM * torch.empty((), dtype=dtype).element_size() + stored = dtype + else: + # FP8 stores one byte per element plus a float32 scale per vector. + vector = HEAD_DIM + 4 + stored = torch.float8_e4m3fn + self.bytes_per_block = 2 * LAYERS * BLOCK * KV_HEADS * vector shape = (max_blocks, LAYERS, BLOCK, KV_HEADS, HEAD_DIM) self.tensors = ( - torch.zeros(shape, dtype=dtype), - torch.zeros(shape, dtype=dtype), + torch.zeros(shape, dtype=stored), + torch.zeros(shape, dtype=stored), ) + self.scale_tensors = None + if with_scales: + scale_shape = (max_blocks, LAYERS, BLOCK, KV_HEADS) + self.scale_tensors = ( + torch.ones(scale_shape, dtype=torch.float32), + torch.ones(scale_shape, dtype=torch.float32), + ) self.max_blocks = max_blocks self.dtype = dtype self.committed_blocks = 0 @@ -386,9 +410,84 @@ def test_commit_measured_capacity_rejects_backing_beyond_the_reservation(): commit_measured_capacity(storage, **commit_args()) -def test_commit_measured_capacity_clamps_to_the_reservation(): +def test_commit_measured_capacity_refuses_to_truncate_to_the_estimate(): + # Decision: the reservation must be sized from the useful cap. If the + # measured capacity would be clamped back to the estimate, startup fails + # rather than shipping a pool that can never grow. storage = FakeStorage(max_blocks=5000, committed=1000) - backed, record = commit_measured_capacity(storage, **commit_args()) + with pytest.raises(RuntimeError, match="reserve the useful cap"): + commit_measured_capacity(storage, **commit_args()) + + +def test_commit_measured_capacity_can_truncate_when_explicitly_allowed(): + storage = FakeStorage(max_blocks=5000, committed=1000) + backed, record = commit_measured_capacity( + storage, **commit_args(allow_reservation_truncation=True) + ) assert record["limited_by"] == "reservation" assert backed == 5000 assert any("exceeds the reservation" in gap for gap in record["coverage_gaps"]) + + +def test_commit_measured_capacity_requires_a_backed_trial_pool(): + storage = FakeStorage(max_blocks=131072, committed=0) + empty = report(free=60 * GIB, total=80 * GIB, blocks=0) + with pytest.raises(ValueError, match="trial pool must back at least one block"): + commit_measured_capacity(storage, **commit_args(report=empty)) + + +def stub_attention(): + return SimpleNamespace( + info={"selected_backend": "stub"}, + capabilities=SimpleNamespace(fp8_kv=True, fp8_kv_static=True), + ) + + +def test_a_mapped_pool_binds_per_vector_fp8_scales(): + storage = FakeStorage(max_blocks=10, committed=4, kv_cache_dtype="fp8_e4m3", with_scales=True) + cache = make_cache( + storage, + num_blocks=4, + kv_cache_dtype="fp8_e4m3", + attention=stub_attention(), + ) + assert cache.key_cache.dtype == torch.float8_e4m3fn + assert cache.key_scale is storage.scale_tensors[0] + assert cache.value_scale is storage.scale_tensors[1] + assert cache._fixed_scale is None + + +def test_a_mapped_pool_accepts_a_fixed_fp8_scale_without_scale_planes(): + storage = FakeStorage(max_blocks=10, committed=4, kv_cache_dtype="fp8_e4m3") + cache = make_cache( + storage, + num_blocks=4, + kv_cache_dtype="fp8_e4m3", + fp8_scale=0.5, + attention=stub_attention(), + ) + assert cache.key_scale is None + assert float(cache._fixed_scale) == 0.5 + + +def test_fp8_over_a_pool_without_scale_support_names_the_hook(): + storage = FakeStorage(max_blocks=10, committed=4, kv_cache_dtype="fp8_e4m3") + with pytest.raises(ValueError, match="scale_tensors"): + make_cache( + storage, + num_blocks=4, + kv_cache_dtype="fp8_e4m3", + attention=stub_attention(), + ) + + +def test_the_pool_must_declare_the_format_it_carries(): + storage = FakeStorage(max_blocks=10, committed=4) # declares "auto" + with pytest.raises(ValueError, match="must declare the format it carries"): + make_cache( + storage, + num_blocks=4, + kv_cache_dtype="fp8_e4m3", + fp8_scale=0.5, + attention=stub_attention(), + ) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 8146f8f8..6a8617b4 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -385,13 +385,20 @@ def _open_torch_planes( def _open_mapped_planes(self, storage: KVCapacitySource, num_blocks: int) -> None: """Bind the K/V planes to a pool that owns their physical mappings. - FP8 KV needs per-vector scales, which the pool does not carry yet, so it - is rejected here rather than writing unquantized bytes into a quantized - cache. Only the pool's backed blocks are exposed; publishing them is the - manager's decision, not the storage's. + The pool must declare the element format it carries. FP8 KV is supported + only through the hook below: a source that carries per-vector scales + exposes ``scale_tensors`` (K first), and one that relies on a fixed + ``fp8_scale`` needs no per-vector storage. Anything else is rejected here + rather than writing unquantized bytes into a quantized cache. Only the + pool's backed blocks are exposed; publishing them is the manager's + decision, not the storage's. """ - if self.kv_cache_dtype != "auto": - raise ValueError("mapped KV storage does not carry FP8 scales yet") + pool_dtype = getattr(storage, "kv_cache_dtype", "auto") + if pool_dtype != self.kv_cache_dtype: + raise ValueError( + f"the pool stores {pool_dtype} KV but the cache was configured for " + f"{self.kv_cache_dtype}; a mapped pool must declare the format it carries" + ) if storage.committed_blocks > storage.max_blocks: raise ValueError( f"storage reports {storage.committed_blocks} committed blocks but " @@ -427,15 +434,48 @@ def _open_mapped_planes(self, storage: KVCapacitySource, num_blocks: int) -> Non self.num_kv_heads, self.head_dim, ) + element_dtype = torch.float8_e4m3fn if self.kv_cache_dtype == "fp8_e4m3" else self.dtype for name, tensor in (("keys", keys), ("values", values)): - if tensor.dtype != self.dtype or tuple(tensor.shape) != shape: + if tensor.dtype != element_dtype or tuple(tensor.shape) != shape: raise ValueError(f"storage {name} do not match the KV layout") self.key_cache, self.value_cache = keys, values - self.key_scale = self.value_scale = None - self._fixed_scale = None + self.key_scale, self.value_scale = self._bind_scale_planes(storage) + self._fixed_scale = ( + torch.tensor(self.fp8_scale, device=self.device, dtype=torch.float32) + if self.fp8_scale is not None + else None + ) self.max_blocks = storage.max_blocks self.committed_blocks = storage.committed_blocks + def _bind_scale_planes(self, storage): + """Return the pool's per-vector FP8 scales, or ``(None, None)``. + + This is the extension point for FP8 KV over a mapped pool. A source that + carries scales exposes ``scale_tensors`` with K first; a source used with + a fixed ``fp8_scale`` needs no per-vector storage. ``committed_bytes`` is + expected to cover the data and the scale pools together. + """ + if self.kv_cache_dtype == "auto" or self.fp8_scale is not None: + return None, None + scales = getattr(storage, "scale_tensors", None) + if scales is None: + raise ValueError( + "mapped FP8 KV needs per-vector scales: expose `scale_tensors` on the " + "capacity source, or configure a fixed fp8_scale" + ) + key_scale, value_scale = scales + scale_shape = ( + storage.max_blocks, + self.num_layers, + self.block_size, + self.num_kv_heads, + ) + for name, tensor in (("key_scale", key_scale), ("value_scale", value_scale)): + if tensor.dtype != torch.float32 or tuple(tensor.shape) != scale_shape: + raise ValueError(f"storage {name} do not match the FP8 scale layout") + return key_scale, value_scale + def publish_capacity(self, blocks: int) -> int: """Grow the allocatable pool after the storage mapped more blocks. diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index f9d0bfec..3e05baaa 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -60,6 +60,11 @@ def measured_kv_capacity( ``safety_bytes`` the fixed reserve. The result is clamped by the logical ``useful_blocks`` and, when the pool cannot grow that far, by ``reservation_blocks``; the binding limit and every margin are recorded. + + ``gpu_memory_utilization`` keeps its existing meaning: a fraction of the + whole device, not of what is free after the baseline. The KV budget is that + cap minus everything already resident, i.e. weights, execution buffers, + captured graphs and mapped blocks. """ for name, value in ( ("bytes_per_block", bytes_per_block), @@ -168,6 +173,7 @@ def commit_measured_capacity( gpu_memory_utilization, transient_bytes, safety_bytes, + allow_reservation_truncation=False, ): """Plan the measured capacity, map it, and return the publishable block count. @@ -175,10 +181,22 @@ def commit_measured_capacity( pool the storage already backs, the plan never exceeds the reservation, and the storage must report at least the planned blocks as backed. The caller publishes exactly the returned count and nothing else. + + Two caller policies are checked here rather than assumed. The trial pool must + already back at least one block, because a mapped pool cannot be built with no + mapping at all. And its reservation must be sized from ``useful_blocks``, not + from the pre-warmup estimate: otherwise the measured capacity is clamped back + to the estimate and the pool can never grow. Pass + ``allow_reservation_truncation=True`` only when that clamp is intended. """ existing = (report or {}).get("num_blocks") if type(existing) is not int or existing < 0: raise ValueError("the startup report must record the current pool as num_blocks") + if existing < 1: + raise ValueError( + "the trial pool must back at least one block; size it from the warmup " + "coverage target before preparing the runtime" + ) if existing > storage.max_blocks: raise ValueError( f"the report describes {existing} blocks but the storage reserves {storage.max_blocks}" @@ -200,6 +218,12 @@ def commit_measured_capacity( existing_blocks=existing, reservation_blocks=storage.max_blocks, ) + if record["limited_by"] == "reservation" and not allow_reservation_truncation: + raise RuntimeError( + f"the measured capacity ({capacity} blocks) exceeds the " + f"{storage.max_blocks}-block reservation, so it would be truncated to the " + "pre-warmup estimate; reserve the useful cap instead of the estimate" + ) backed = storage.commit(capacity) if type(backed) is not int or backed < capacity: raise RuntimeError(f"storage reported {backed} backed blocks after committing {capacity}")