diff --git a/tests/test_measured_capacity.py b/tests/test_measured_capacity.py new file mode 100644 index 00000000..3f4df296 --- /dev/null +++ b/tests/test_measured_capacity.py @@ -0,0 +1,493 @@ +"""Measured KV capacity (RFC #192 step 2 handoff) and the publish gate. + +``measured_kv_capacity`` consumes the step-1 preparation report; the expected +values below are hand-computed from the KV layout and the stated margins, not +from the implementation. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.core.memory import ( + commit_measured_capacity, + measured_kv_capacity, + transient_allowance, +) + +MIB = 1 << 20 +GIB = 1024 * MIB +LAYERS, KV_HEADS, HEAD_DIM, BLOCK = 2, 1, 4, 2 + + +def report(*, free, total, gaps=(), blocks=None): + """One drained, post-cleanup sample as ``prepare_runtime`` reports it.""" + sample = dict( + phases=[ + dict( + name="prepared", + driver_free_bytes=free, + device_total_bytes=total, + allocated_bytes=0, + reserved_bytes=0, + ) + ], + coverage_gaps=tuple(gaps), + ) + if blocks is not None: + sample["num_blocks"] = blocks + return sample + + +def plan(**overrides): + parameters = dict( + report=report(free=60 * GIB, total=80 * GIB), + bytes_per_block=3 * MIB, + useful_blocks=10**6, + gpu_memory_utilization=0.9, + transient_bytes=4 * GIB, + safety_bytes=256 * MIB, + ) + parameters.update(overrides) + return measured_kv_capacity(**parameters) + + +def test_blocks_follow_the_measured_budget_without_double_counting(): + blocks, record = plan() + # resident = 80 - 60 = 20 GiB; cap = 72 GiB; remaining = min(60, 72 - 20) = 52 GiB + # budget = 52 GiB - 4 GiB - 256 MiB = 48896 MiB; 48896 // 3 = 16298 blocks + assert (record["resident_bytes"], record["utilization_cap_bytes"]) == (20 * GIB, int(72 * GIB)) + assert record["remaining_instance_bytes"] == 52 * GIB + assert record["kv_budget_bytes"] == 53248 * MIB - 4096 * MIB - 256 * MIB + assert blocks == 16298 + assert record["source"] == "measured_warmup" + assert blocks * record["bytes_per_block"] <= record["kv_budget_bytes"] + + +def test_driver_free_memory_is_bounded_by_the_utilization_cap(): + # Only 1 GiB is resident, so the 0.5 cap (40 GiB) binds before free memory. + blocks, record = plan( + report=report(free=79 * GIB, total=80 * GIB), + gpu_memory_utilization=0.5, + bytes_per_block=MIB, + transient_bytes=0, + safety_bytes=0, + ) + assert record["remaining_instance_bytes"] == 39 * GIB + assert blocks == 39 * 1024 + + +def test_each_margin_reduces_capacity_exactly(): + free = 20 * GIB + base, _ = plan( + report=report(free=free, total=80 * GIB), + transient_bytes=0, + safety_bytes=0, + gpu_memory_utilization=1.0, + ) + transient, _ = plan( + report=report(free=free, total=80 * GIB), + transient_bytes=1024 * 3 * MIB, + safety_bytes=0, + gpu_memory_utilization=1.0, + ) + safety, _ = plan( + report=report(free=free, total=80 * GIB), + transient_bytes=0, + safety_bytes=1024 * 3 * MIB, + gpu_memory_utilization=1.0, + ) + assert base - transient == 1024 + assert base - safety == 1024 + + +def test_useful_blocks_cap_the_measured_budget(): + blocks, record = plan(useful_blocks=100) + assert blocks == 100 + assert record["max_useful_blocks"] == 100 + + +def test_coverage_gaps_travel_with_the_plan(): + gaps = ("contexts beyond the reported workloads retain the estimated reserve",) + _, record = plan(report=report(free=60 * GIB, total=80 * GIB, gaps=gaps)) + assert record["coverage_gaps"] == gaps + + +def test_a_budget_below_one_block_is_rejected_with_the_margins(): + with pytest.raises(ValueError, match="remaining=.*transient=.*safety=.*block="): + plan( + report=report(free=8 * MIB, total=80 * GIB), + transient_bytes=0, + safety_bytes=0, + ) + + +@pytest.mark.parametrize( + "bad_report", + [None, {}, dict(phases=[]), dict(phases=[dict(driver_free_bytes=1)])], +) +def test_a_missing_or_incomplete_report_is_rejected(bad_report): + with pytest.raises(ValueError, match="report|phase"): + plan(report=bad_report) + + +@pytest.mark.parametrize( + "overrides, message", + [ + (dict(bytes_per_block=0), "bytes_per_block"), + (dict(useful_blocks=0), "useful_blocks"), + (dict(transient_bytes=-1), "transient_bytes"), + (dict(safety_bytes=-1), "safety_bytes"), + (dict(gpu_memory_utilization=0.0), "gpu_memory_utilization"), + ], +) +def test_invalid_inputs_are_rejected(overrides, message): + with pytest.raises(ValueError, match=message): + plan(**overrides) + + +class FakeStorage: + """Mapped storage stand-in: only ``committed_blocks`` has backing.""" + + def __init__( + self, + *, + max_blocks, + committed=0, + dtype=torch.float32, + kv_cache_dtype="auto", + with_scales=False, + ): + self.kv_cache_dtype = kv_cache_dtype + if kv_cache_dtype == "auto": + vector = HEAD_DIM * torch.empty((), dtype=dtype).element_size() + stored = dtype + else: + # FP8 stores one byte per element plus a float32 scale per vector. + vector = HEAD_DIM + 4 + stored = torch.float8_e4m3fn + self.bytes_per_block = 2 * LAYERS * BLOCK * KV_HEADS * vector + shape = (max_blocks, LAYERS, BLOCK, KV_HEADS, HEAD_DIM) + self.tensors = ( + torch.zeros(shape, dtype=stored), + torch.zeros(shape, dtype=stored), + ) + self.scale_tensors = None + if with_scales: + scale_shape = (max_blocks, LAYERS, BLOCK, KV_HEADS) + self.scale_tensors = ( + torch.ones(scale_shape, dtype=torch.float32), + torch.ones(scale_shape, dtype=torch.float32), + ) + self.max_blocks = max_blocks + self.dtype = dtype + self.committed_blocks = 0 + self.committed_bytes = 0 + self.commit(committed) + + def commit(self, blocks): + self.committed_blocks = max(self.committed_blocks, blocks) + self.committed_bytes = self.committed_blocks * self.bytes_per_block + return self.committed_blocks + + +def make_cache(storage=None, num_blocks=4, **kwargs): + parameters = dict( + num_layers=LAYERS, + num_kv_heads=KV_HEADS, + head_dim=HEAD_DIM, + num_blocks=num_blocks, + block_size=BLOCK, + max_loops=1, + ) + parameters.update(kwargs) + return KVCacheManager(**parameters, storage=storage) + + +def test_storage_backed_cache_starts_at_its_committed_capacity(): + storage = FakeStorage(max_blocks=10, committed=4) + cache = make_cache(storage, num_blocks=4) + assert (cache.published_blocks, cache.committed_blocks, cache.max_blocks) == (4, 4, 10) + assert cache.num_blocks == 4 + assert cache.num_free_blocks == 4 + assert cache.key_cache.shape == storage.tensors[0].shape + + +def test_publish_requires_committed_memory_and_grows_the_free_pool(): + storage = FakeStorage(max_blocks=10, committed=2) + cache = make_cache(storage, num_blocks=2) + assert cache.allocate("a", 4) # two published blocks + assert cache.allocate("b", 2) is False + with pytest.raises(ValueError, match="committed 2"): + cache.publish_capacity(8) + assert (cache.published_blocks, cache.num_free_blocks) == (2, 0) + + storage.commit(10) + assert cache.publish_capacity(10) == 10 + assert cache.num_free_blocks == 8 + assert cache.allocate("b", 2) + blocks = cache.get_block_table("b", 0) + assert set(blocks) & set(range(2, 10)), "growth must hand out the newly backed blocks" + key = torch.full((1, KV_HEADS, HEAD_DIM), 3.0) + cache.write(0, ["b"], [0], [0], key, key + 1) + assert torch.equal(cache.key_cache[blocks[0], 0, 0], key[0]) + + +def test_publish_is_monotonic_and_bounded(): + storage = FakeStorage(max_blocks=10, committed=10) + cache = make_cache(storage, num_blocks=2) + assert cache.publish_capacity(6) == 6 + with pytest.raises(ValueError, match="only grow"): + cache.publish_capacity(4) + with pytest.raises(ValueError, match="reservation holds"): + cache.publish_capacity(11) + assert cache.publish_capacity(6) == 6 # idempotent + assert cache.num_blocks == 6 + + +def test_publish_requires_storage(): + cache = make_cache(None) + with pytest.raises(RuntimeError, match="mapped storage"): + cache.publish_capacity(8) + + +def test_plain_cache_keeps_fixed_capacity(): + cache = make_cache(None, num_blocks=6) + assert cache.storage is None + assert (cache.max_blocks, cache.committed_blocks, cache.published_blocks) == (6, 6, 6) + + +def test_storage_must_be_committed_before_the_cache_is_built(): + storage = FakeStorage(max_blocks=10, committed=2) + with pytest.raises(ValueError, match="commit the pool"): + make_cache(storage, num_blocks=4) + + +def test_storage_must_cover_the_requested_blocks_and_layout(): + coarse = FakeStorage(max_blocks=2, committed=2) + with pytest.raises(ValueError, match="reserves"): + make_cache(coarse, num_blocks=4) + wrong_dtype = FakeStorage(max_blocks=10, committed=10, dtype=torch.bfloat16) + with pytest.raises(ValueError, match="do not match the KV layout"): + make_cache(wrong_dtype, num_blocks=4) + + +def test_existing_bootstrap_blocks_are_counted_once_in_the_total(): + # The bootstrap prefix is already mapped, so it sits outside `free` and must + # extend the measured delta rather than being sized from free again. + capacity, record = plan(existing_blocks=100) + assert record["additional_blocks"] == 16298 + assert capacity == 16398 + assert record["existing_blocks"] == 100 + # Only the new part spends budget. + assert (capacity - 100) * record["bytes_per_block"] <= record["kv_budget_bytes"] + + +def test_bootstrap_only_capacity_is_usable_without_growth(): + capacity, record = plan( + report=report(free=8 * MIB, total=80 * GIB), + existing_blocks=8, + transient_bytes=0, + safety_bytes=0, + ) + assert capacity == 8 + assert record["additional_blocks"] == 0 + assert record["limited_by"] == "budget" + + +def test_the_reservation_truncates_the_measured_capacity(): + capacity, record = plan(reservation_blocks=1000) + assert capacity == 1000 + assert record["limited_by"] == "reservation" + assert any("exceeds the reservation" in gap for gap in record["coverage_gaps"]) + + +def test_the_useful_block_cap_binds_before_the_reservation(): + capacity, record = plan(useful_blocks=100, reservation_blocks=1000) + assert capacity == 100 + assert record["limited_by"] == "useful_blocks" + assert not any("exceeds the reservation" in gap for gap in record["coverage_gaps"]) + + +def test_coverage_gaps_reject_a_zero_transient_allowance(): + gaps = ("random block speculation retains the estimated transient reserve",) + with pytest.raises(ValueError, match="coverage gaps require an explicit transient"): + plan( + report=report(free=60 * GIB, total=80 * GIB, gaps=gaps), + transient_bytes=0, + safety_bytes=0, + ) + # The same report is accepted once an allowance is stated, and the gap travels with it. + _, record = plan(report=report(free=60 * GIB, total=80 * GIB, gaps=gaps)) + assert record["coverage_gaps"] == gaps + + +def test_storage_may_not_report_more_committed_blocks_than_it_reserves(): + storage = FakeStorage(max_blocks=4, committed=4) + storage.committed_blocks = 5 # a storage-side accounting bug + with pytest.raises(ValueError, match="reserves only"): + make_cache(storage, num_blocks=4) + + +def test_transient_allowance_takes_the_larger_of_measured_and_estimate(): + phases = [ + dict(name="initialized", allocated_bytes=100 * MIB, peak_allocated_bytes=400 * MIB), + dict(name="prepared", allocated_bytes=300 * MIB, peak_allocated_bytes=500 * MIB), + ] + value, record = transient_allowance({"phases": phases}, estimate_bytes=64 * MIB) + assert record["measured_transient_bytes"] == 300 * MIB # max(300, 500 - 300) + assert value == 300 * MIB + assert record["sampled_phases"] == 2 + + value, _ = transient_allowance({"phases": phases}, estimate_bytes=512 * MIB) + assert value == 512 * MIB # an unexercised path keeps the estimate + + value, _ = transient_allowance({"phases": phases}, estimate_bytes=0, overlap_factor=2) + assert value == 600 * MIB # a serial peak does not bound a concurrent envelope + + +def test_transient_allowance_ignores_phases_without_peak_samples(): + value, record = transient_allowance({"phases": [dict(name="cpu")]}, estimate_bytes=MIB) + assert record["sampled_phases"] == 0 + assert value == MIB + + +@pytest.mark.parametrize( + "kwargs", [dict(estimate_bytes=-1), dict(estimate_bytes=0, overlap_factor=0)] +) +def test_transient_allowance_rejects_invalid_inputs(kwargs): + with pytest.raises(ValueError): + transient_allowance({"phases": []}, **kwargs) + + +def commit_args(**kwargs): + """Arguments for ``commit_measured_capacity`` on the 60/80 GiB sample.""" + options = dict( + report=report(free=60 * GIB, total=80 * GIB, blocks=1000), + bytes_per_block=3 * MIB, + useful_blocks=131072, + gpu_memory_utilization=0.9, + transient_bytes=4 * GIB, + safety_bytes=256 * MIB, + ) + options.update(kwargs) + return options + + +def test_commit_measured_capacity_maps_the_plan_and_returns_backed_blocks(): + storage = FakeStorage(max_blocks=131072, committed=1000) + backed, record = commit_measured_capacity(storage, **commit_args()) + assert backed == 1000 + 16298 + assert storage.committed_blocks == backed + assert record["committed_blocks"] == backed + assert record["storage"] == "mapped" + assert record["limited_by"] == "budget" + + +def test_commit_measured_capacity_requires_the_report_to_describe_this_pool(): + storage = FakeStorage(max_blocks=131072, committed=1000) + drifted = report(free=60 * GIB, total=80 * GIB, blocks=999) + with pytest.raises(ValueError, match="storage backs"): + commit_measured_capacity(storage, **commit_args(report=drifted)) + no_pool = report(free=60 * GIB, total=80 * GIB) + with pytest.raises(ValueError, match="num_blocks"): + commit_measured_capacity(storage, **commit_args(report=no_pool)) + + +def test_commit_measured_capacity_rejects_a_storage_that_under_commits(): + storage = FakeStorage(max_blocks=131072, committed=1000) + storage.commit = lambda blocks: storage.committed_blocks + with pytest.raises(RuntimeError, match="after committing"): + commit_measured_capacity(storage, **commit_args()) + + +def test_commit_measured_capacity_rejects_backing_beyond_the_reservation(): + storage = FakeStorage(max_blocks=131072, committed=1000) + storage.commit = lambda blocks: storage.max_blocks + 1 + with pytest.raises(RuntimeError, match="beyond its"): + commit_measured_capacity(storage, **commit_args()) + + +def test_commit_measured_capacity_refuses_to_truncate_to_the_estimate(): + # Decision: the reservation must be sized from the useful cap. If the + # measured capacity would be clamped back to the estimate, startup fails + # rather than shipping a pool that can never grow. + storage = FakeStorage(max_blocks=5000, committed=1000) + with pytest.raises(RuntimeError, match="reserve the useful cap"): + commit_measured_capacity(storage, **commit_args()) + + +def test_commit_measured_capacity_can_truncate_when_explicitly_allowed(): + storage = FakeStorage(max_blocks=5000, committed=1000) + backed, record = commit_measured_capacity( + storage, **commit_args(allow_reservation_truncation=True) + ) + assert record["limited_by"] == "reservation" + assert backed == 5000 + assert any("exceeds the reservation" in gap for gap in record["coverage_gaps"]) + + +def test_commit_measured_capacity_requires_a_backed_trial_pool(): + storage = FakeStorage(max_blocks=131072, committed=0) + empty = report(free=60 * GIB, total=80 * GIB, blocks=0) + with pytest.raises(ValueError, match="trial pool must back at least one block"): + commit_measured_capacity(storage, **commit_args(report=empty)) + + +def stub_attention(): + return SimpleNamespace( + info={"selected_backend": "stub"}, + capabilities=SimpleNamespace(fp8_kv=True, fp8_kv_static=True), + ) + + +def test_a_mapped_pool_binds_per_vector_fp8_scales(): + storage = FakeStorage(max_blocks=10, committed=4, kv_cache_dtype="fp8_e4m3", with_scales=True) + cache = make_cache( + storage, + num_blocks=4, + kv_cache_dtype="fp8_e4m3", + attention=stub_attention(), + ) + assert cache.key_cache.dtype == torch.float8_e4m3fn + assert cache.key_scale is storage.scale_tensors[0] + assert cache.value_scale is storage.scale_tensors[1] + assert cache._fixed_scale is None + + +def test_a_mapped_pool_accepts_a_fixed_fp8_scale_without_scale_planes(): + storage = FakeStorage(max_blocks=10, committed=4, kv_cache_dtype="fp8_e4m3") + cache = make_cache( + storage, + num_blocks=4, + kv_cache_dtype="fp8_e4m3", + fp8_scale=0.5, + attention=stub_attention(), + ) + assert cache.key_scale is None + assert float(cache._fixed_scale) == 0.5 + + +def test_fp8_over_a_pool_without_scale_support_names_the_hook(): + storage = FakeStorage(max_blocks=10, committed=4, kv_cache_dtype="fp8_e4m3") + with pytest.raises(ValueError, match="scale_tensors"): + make_cache( + storage, + num_blocks=4, + kv_cache_dtype="fp8_e4m3", + attention=stub_attention(), + ) + + +def test_the_pool_must_declare_the_format_it_carries(): + storage = FakeStorage(max_blocks=10, committed=4) # declares "auto" + with pytest.raises(ValueError, match="must declare the format it carries"): + make_cache( + storage, + num_blocks=4, + kv_cache_dtype="fp8_e4m3", + fp8_scale=0.5, + attention=stub_attention(), + ) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index c6187d77..6a8617b4 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -18,6 +18,7 @@ from copy import deepcopy from dataclasses import dataclass, field from numbers import Integral +from typing import Protocol import torch @@ -166,6 +167,23 @@ class KVSnapshot: fp8_scale: float | None = None +class KVCapacitySource(Protocol): + """Storage that maps physical memory behind one fixed address range. + + ``tensors`` holds the K and V planes in that order and spans ``max_blocks``; + only the first ``committed_blocks`` are backed. ``commit(blocks)`` maps more + and returns how many blocks are fully backed. Release is by reference, as + with ``torch.empty``: the storage owner decides, not this manager. + """ + + max_blocks: int + committed_blocks: int + committed_bytes: int + tensors: tuple[torch.Tensor, ...] + + def commit(self, blocks: int) -> int: ... + + class KVCacheManager: """Own a fixed physical page pool shared by request/depth allocations. @@ -194,6 +212,7 @@ def __init__( attention: AttentionBackend | None = None, kv_cache_dtype: str = "auto", fp8_scale: float | None = None, + storage: KVCapacitySource | None = None, ): for name, value in ( ("num_layers", num_layers), @@ -252,30 +271,23 @@ def __init__( ) self.attention_info = self.attention.info self.backend = self.attention_info["selected_backend"] - capability = "fp8_kv" if fp8_scale is None else "fp8_kv_static" - if kv_cache_dtype != "auto" and not getattr(self.attention.capabilities, capability): - raise ValueError( - f"the {self.attention_info['backend']} backend does not read FP8 KV " - f"with {'per-vector' if fp8_scale is None else 'fixed'} scales" + self._storage = storage + self._watermark_ratio = watermark_ratio + if storage is None: + self._open_torch_planes( + num_blocks=num_blocks, + block_size=block_size, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + dtype=dtype, + kv_cache_dtype=kv_cache_dtype, + fp8_scale=fp8_scale, ) - shape = (num_blocks, num_layers, block_size, num_kv_heads, head_dim) - storage = torch.float8_e4m3fn if kv_cache_dtype == "fp8_e4m3" else dtype - self.key_cache = torch.empty(shape, device=self.device, dtype=storage) - self.value_cache = torch.empty_like(self.key_cache) - # One scale per stored vector; None for an unquantized pool. - self.key_scale = self.value_scale = None - if kv_cache_dtype == "fp8_e4m3": - # Imported fixed-scale pages use the same local scale as locally written pages. - scale = 1.0 if fp8_scale is None else fp8_scale - self.key_scale = torch.full(shape[:-1], scale, device=self.device, dtype=torch.float32) - self.value_scale = torch.full_like(self.key_scale, scale) - # Keep the existing per-vector storage/lifecycle for this prototype. FA - # borrows the immutable scalar; Torch/Triton continue reading the scale pool. - self._fixed_scale = ( - torch.tensor(fp8_scale, device=self.device, dtype=torch.float32) - if fp8_scale is not None - else None - ) + else: + self._open_mapped_planes(storage, num_blocks) + # Requests only ever receive published blocks, and publishing requires + # committed physical memory, so published <= committed <= max holds. + self.published_blocks = num_blocks # Resolve an implicit CUDA index to the actual storage device once. self.device = self.key_cache.device self._free_blocks = list(reversed(range(num_blocks))) @@ -291,7 +303,7 @@ def __init__( self.enable_prefix_caching = enable_prefix_caching self.incremental_allocation = incremental_allocation # Full allocation already reserves each request's maximum KV capacity. - self.watermark_blocks = int(watermark_ratio * num_blocks) if incremental_allocation else 0 + self.watermark_blocks = self._watermark(num_blocks) self._refs = [0] * num_blocks self._prefixes = OrderedDict() self._pending_prefixes = [] @@ -324,6 +336,177 @@ def bytes_per_block(self) -> int: self.kv_cache_dtype, ) + @property + def storage(self) -> KVCapacitySource | None: + return self._storage + + def _watermark(self, blocks: int) -> int: + return int(self._watermark_ratio * blocks) if self.incremental_allocation else 0 + + def _open_torch_planes( + self, + *, + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_dim: int, + dtype: torch.dtype, + kv_cache_dtype: str, + fp8_scale: float | None, + ) -> None: + """Allocate the K/V planes with the PyTorch allocator, including FP8 scales.""" + capability = "fp8_kv" if fp8_scale is None else "fp8_kv_static" + if kv_cache_dtype != "auto" and not getattr(self.attention.capabilities, capability): + raise ValueError( + f"the {self.attention_info['backend']} backend does not read FP8 KV " + f"with {'per-vector' if fp8_scale is None else 'fixed'} scales" + ) + shape = (num_blocks, self.num_layers, block_size, num_kv_heads, head_dim) + storage_dtype = torch.float8_e4m3fn if kv_cache_dtype == "fp8_e4m3" else dtype + self.key_cache = torch.empty(shape, device=self.device, dtype=storage_dtype) + self.value_cache = torch.empty_like(self.key_cache) + # One scale per stored vector; None for an unquantized pool. + self.key_scale = self.value_scale = None + if kv_cache_dtype == "fp8_e4m3": + # Imported fixed-scale pages use the same local scale as locally written pages. + scale = 1.0 if fp8_scale is None else fp8_scale + self.key_scale = torch.full(shape[:-1], scale, device=self.device, dtype=torch.float32) + self.value_scale = torch.full_like(self.key_scale, scale) + # Keep the existing per-vector storage/lifecycle for this prototype. FA + # borrows the immutable scalar; Torch/Triton continue reading the scale pool. + self._fixed_scale = ( + torch.tensor(fp8_scale, device=self.device, dtype=torch.float32) + if fp8_scale is not None + else None + ) + self.max_blocks = num_blocks + self.committed_blocks = num_blocks + + def _open_mapped_planes(self, storage: KVCapacitySource, num_blocks: int) -> None: + """Bind the K/V planes to a pool that owns their physical mappings. + + The pool must declare the element format it carries. FP8 KV is supported + only through the hook below: a source that carries per-vector scales + exposes ``scale_tensors`` (K first), and one that relies on a fixed + ``fp8_scale`` needs no per-vector storage. Anything else is rejected here + rather than writing unquantized bytes into a quantized cache. Only the + pool's backed blocks are exposed; publishing them is the manager's + decision, not the storage's. + """ + pool_dtype = getattr(storage, "kv_cache_dtype", "auto") + if pool_dtype != self.kv_cache_dtype: + raise ValueError( + f"the pool stores {pool_dtype} KV but the cache was configured for " + f"{self.kv_cache_dtype}; a mapped pool must declare the format it carries" + ) + if storage.committed_blocks > storage.max_blocks: + raise ValueError( + f"storage reports {storage.committed_blocks} committed blocks but " + f"reserves only {storage.max_blocks}" + ) + expected = kv_bytes_per_block( + self.num_layers, + self.block_size, + self.num_kv_heads, + self.head_dim, + torch.empty((), dtype=self.dtype).element_size(), + self.kv_cache_dtype, + ) + if storage.max_blocks < num_blocks: + raise ValueError( + f"storage reserves {storage.max_blocks} blocks but {num_blocks} were requested" + ) + if storage.committed_blocks < num_blocks: + raise ValueError( + f"storage committed {storage.committed_blocks} blocks but {num_blocks} " + "were requested; commit the pool before building the cache" + ) + if storage.committed_bytes < num_blocks * expected: + raise ValueError( + f"storage reports {storage.committed_bytes} committed bytes but " + f"{num_blocks} blocks need {num_blocks * expected}" + ) + keys, values = storage.tensors + shape = ( + storage.max_blocks, + self.num_layers, + self.block_size, + self.num_kv_heads, + self.head_dim, + ) + element_dtype = torch.float8_e4m3fn if self.kv_cache_dtype == "fp8_e4m3" else self.dtype + for name, tensor in (("keys", keys), ("values", values)): + if tensor.dtype != element_dtype or tuple(tensor.shape) != shape: + raise ValueError(f"storage {name} do not match the KV layout") + self.key_cache, self.value_cache = keys, values + self.key_scale, self.value_scale = self._bind_scale_planes(storage) + self._fixed_scale = ( + torch.tensor(self.fp8_scale, device=self.device, dtype=torch.float32) + if self.fp8_scale is not None + else None + ) + self.max_blocks = storage.max_blocks + self.committed_blocks = storage.committed_blocks + + def _bind_scale_planes(self, storage): + """Return the pool's per-vector FP8 scales, or ``(None, None)``. + + This is the extension point for FP8 KV over a mapped pool. A source that + carries scales exposes ``scale_tensors`` with K first; a source used with + a fixed ``fp8_scale`` needs no per-vector storage. ``committed_bytes`` is + expected to cover the data and the scale pools together. + """ + if self.kv_cache_dtype == "auto" or self.fp8_scale is not None: + return None, None + scales = getattr(storage, "scale_tensors", None) + if scales is None: + raise ValueError( + "mapped FP8 KV needs per-vector scales: expose `scale_tensors` on the " + "capacity source, or configure a fixed fp8_scale" + ) + key_scale, value_scale = scales + scale_shape = ( + storage.max_blocks, + self.num_layers, + self.block_size, + self.num_kv_heads, + ) + for name, tensor in (("key_scale", key_scale), ("value_scale", value_scale)): + if tensor.dtype != torch.float32 or tuple(tensor.shape) != scale_shape: + raise ValueError(f"storage {name} do not match the FP8 scale layout") + return key_scale, value_scale + + def publish_capacity(self, blocks: int) -> int: + """Grow the allocatable pool after the storage mapped more blocks. + + The K/V tensors keep their address, shape and stride; only the free pool + and the published count grow. Uncommitted blocks are never published, so + no allocation can reference unmapped memory. + """ + if self._storage is None: + raise RuntimeError("capacity publishing requires mapped storage") + if type(blocks) is not int or blocks < self.published_blocks: + raise ValueError("published capacity can only grow") + if blocks > self.max_blocks: + raise ValueError( + f"cannot publish {blocks} blocks; the reservation holds {self.max_blocks}" + ) + # Re-read the pool: a commit may have happened since the last publish. + self.committed_blocks = self._storage.committed_blocks + if blocks > self.committed_blocks: + raise ValueError( + f"cannot publish {blocks} blocks; the storage committed {self.committed_blocks}" + ) + if blocks == self.published_blocks: + return blocks + self._refs.extend([0] * (blocks - self.published_blocks)) + # _claim pops from the tail, so later blocks follow in ascending order. + self._free_blocks.extend(reversed(range(self.published_blocks, blocks))) + self.published_blocks = blocks + self.num_blocks = blocks + self.watermark_blocks = self._watermark(blocks) + return blocks + def required_blocks(self, max_tokens: int) -> int: if not isinstance(max_tokens, Integral) or isinstance(max_tokens, bool) or max_tokens <= 0: raise ValueError("max_tokens must be a positive integer") diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index 1c52c16c..3e05baaa 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -19,6 +19,223 @@ def budget_blocks(available_bytes, bytes_per_block): return blocks +def measurement_budget(report): + """Driver free/total bytes and coverage gaps from a startup preparation report. + + ``report`` is step 1 of RFC #192 (``StageEngine.prepare_runtime``): boundary + samples taken after draining, with the allocator cache retained. Only the + last sample sizes capacity, because it is the state after warmup and capture. + """ + if not report: + raise ValueError("a measured KV plan requires a startup preparation report") + phases = report.get("phases") or () + if not phases: + raise ValueError("the startup report contains no phase samples") + final = phases[-1] + free, total = final.get("driver_free_bytes"), final.get("device_total_bytes") + if type(free) is not int or type(total) is not int or total <= 0 or free < 0: + raise ValueError("the final startup phase lacks valid driver free/total bytes") + return free, total, tuple(report.get("coverage_gaps") or ()) + + +def measured_kv_capacity( + *, + report, + bytes_per_block, + useful_blocks, + gpu_memory_utilization, + transient_bytes, + safety_bytes, + existing_blocks=0, + reservation_blocks=None, +): + """Total blocks the pool may hold after warmup and capture. + + ``free`` memory already excludes weights, execution buffers, captured graphs + and anything already mapped, including ``existing_blocks`` (the bootstrap + prefix). The measured capacity is therefore additional, and the cumulative + ``commit(total)`` of a storage maps only the new part. + + ``transient_bytes`` covers execution above that persistent baseline and + ``safety_bytes`` the fixed reserve. The result is clamped by the logical + ``useful_blocks`` and, when the pool cannot grow that far, by + ``reservation_blocks``; the binding limit and every margin are recorded. + + ``gpu_memory_utilization`` keeps its existing meaning: a fraction of the + whole device, not of what is free after the baseline. The KV budget is that + cap minus everything already resident, i.e. weights, execution buffers, + captured graphs and mapped blocks. + """ + for name, value in ( + ("bytes_per_block", bytes_per_block), + ("useful_blocks", useful_blocks), + ): + if type(value) is not int or value < 1: + raise ValueError(f"{name} must be a positive integer") + for name, value in ( + ("transient_bytes", transient_bytes), + ("safety_bytes", safety_bytes), + ("existing_blocks", existing_blocks), + ): + if type(value) is not int or value < 0: + raise ValueError(f"{name} must be a nonnegative integer") + if reservation_blocks is not None and ( + type(reservation_blocks) is not int or reservation_blocks < 1 + ): + raise ValueError("reservation_blocks must be a positive integer") + if not 0 < gpu_memory_utilization <= 1: + raise ValueError("gpu_memory_utilization must be in (0, 1]") + free, total, gaps = measurement_budget(report) + if gaps and not transient_bytes: + # The preparation reported paths it did not exercise. Sizing them as if + # measured would publish a pool that cannot serve them. + raise ValueError(f"coverage gaps require an explicit transient allowance: {gaps[0]}") + resident = total - free + # What the utilization cap still allows, bounded by measured free memory. + remaining = min(free, int(total * gpu_memory_utilization) - resident) + budget = remaining - transient_bytes - safety_bytes + additional = max(0, budget // bytes_per_block) + capacity = existing_blocks + additional + if capacity < 1: + raise ValueError( + "no KV capacity fits the measured budget: " + f"remaining={remaining} transient={transient_bytes} " + f"safety={safety_bytes} block={bytes_per_block}" + ) + limit = "budget" + if capacity > useful_blocks: + capacity, limit = useful_blocks, "useful_blocks" + if reservation_blocks is not None and capacity > reservation_blocks: + capacity, limit = reservation_blocks, "reservation" + gaps = ( + *gaps, + "measured capacity exceeds the reservation; committed capacity truncated", + ) + return capacity, dict( + source="measured_warmup", + bytes_per_block=bytes_per_block, + driver_free_bytes=free, + device_total_bytes=total, + resident_bytes=resident, + utilization_cap_bytes=int(total * gpu_memory_utilization), + remaining_instance_bytes=remaining, + transient_bytes=transient_bytes, + safety_bytes=safety_bytes, + kv_budget_bytes=budget, + existing_blocks=existing_blocks, + additional_blocks=additional, + max_useful_blocks=useful_blocks, + reservation_blocks=reservation_blocks, + limited_by=limit, + coverage_gaps=gaps, + ) + + +def transient_allowance(report, *, estimate_bytes, overlap_factor=1): + """Allowance for execution above the persistent baseline of a phase. + + A phase records its end state and its own peak, so ``peak - allocated`` is + what that phase needed on top of what it kept. The largest such delta is + measured; ``estimate_bytes`` covers paths the preparation did not exercise + and must stay in force. ``overlap_factor`` covers supported concurrency, + because a serial peak understates a concurrent envelope. The larger of the + two is returned, so an unmeasured path is never sized as if measured. + """ + if type(estimate_bytes) is not int or estimate_bytes < 0: + raise ValueError("estimate_bytes must be a nonnegative integer") + if type(overlap_factor) is not int or overlap_factor < 1: + raise ValueError("overlap_factor must be an integer of at least 1") + phases = (report or {}).get("phases") or () + measured = 0 + sampled = 0 + for phase in phases: + peak, allocated = phase.get("peak_allocated_bytes"), phase.get("allocated_bytes") + if type(peak) is not int or type(allocated) is not int: + continue + sampled += 1 + measured = max(measured, peak - allocated) + value = max(measured * overlap_factor, estimate_bytes) + return value, dict( + transient_bytes=value, + measured_transient_bytes=measured, + estimate_bytes=estimate_bytes, + overlap_factor=overlap_factor, + sampled_phases=sampled, + ) + + +def commit_measured_capacity( + storage, + *, + report, + bytes_per_block, + useful_blocks, + gpu_memory_utilization, + transient_bytes, + safety_bytes, + allow_reservation_truncation=False, +): + """Plan the measured capacity, map it, and return the publishable block count. + + Enforces the preconditions of the agreed ordering: the report describes the + pool the storage already backs, the plan never exceeds the reservation, and + the storage must report at least the planned blocks as backed. The caller + publishes exactly the returned count and nothing else. + + Two caller policies are checked here rather than assumed. The trial pool must + already back at least one block, because a mapped pool cannot be built with no + mapping at all. And its reservation must be sized from ``useful_blocks``, not + from the pre-warmup estimate: otherwise the measured capacity is clamped back + to the estimate and the pool can never grow. Pass + ``allow_reservation_truncation=True`` only when that clamp is intended. + """ + existing = (report or {}).get("num_blocks") + if type(existing) is not int or existing < 0: + raise ValueError("the startup report must record the current pool as num_blocks") + if existing < 1: + raise ValueError( + "the trial pool must back at least one block; size it from the warmup " + "coverage target before preparing the runtime" + ) + if existing > storage.max_blocks: + raise ValueError( + f"the report describes {existing} blocks but the storage reserves {storage.max_blocks}" + ) + if storage.committed_blocks != existing: + # The pool changed after the sample, so the measured free memory no + # longer describes this storage. + raise ValueError( + f"the report describes {existing} blocks but the storage backs " + f"{storage.committed_blocks}" + ) + capacity, record = measured_kv_capacity( + report=report, + bytes_per_block=bytes_per_block, + useful_blocks=useful_blocks, + gpu_memory_utilization=gpu_memory_utilization, + transient_bytes=transient_bytes, + safety_bytes=safety_bytes, + existing_blocks=existing, + reservation_blocks=storage.max_blocks, + ) + if record["limited_by"] == "reservation" and not allow_reservation_truncation: + raise RuntimeError( + f"the measured capacity ({capacity} blocks) exceeds the " + f"{storage.max_blocks}-block reservation, so it would be truncated to the " + "pre-warmup estimate; reserve the useful cap instead of the estimate" + ) + backed = storage.commit(capacity) + if type(backed) is not int or backed < capacity: + raise RuntimeError(f"storage reported {backed} backed blocks after committing {capacity}") + if backed > storage.max_blocks: + raise RuntimeError( + f"storage reported {backed} backed blocks beyond its " + f"{storage.max_blocks}-block reservation" + ) + record.update(committed_blocks=backed, storage="mapped") + return backed, record + + def graph_memory_reserve(model, scheduler, execution, speculative=None): """Headroom per pool plus retained speculative coda logits across all entries.