diff --git a/docs/async-speculative.md b/docs/async-speculative.md index 6a003d1a..b03a42a6 100644 --- a/docs/async-speculative.md +++ b/docs/async-speculative.md @@ -983,7 +983,11 @@ combination with compatible attention backends such as Triton and FA4. The asynchronous pipeline uses the same compute/copy streams and two leased result buffers as asynchronous block speculation. The compute stream advances or rejects token slots; the copy stream transfers compact results to pinned host storage. -Collecting an older step never rewinds GPU state. Inactive slots have masked KV +Collecting an older step never rewinds GPU state. The token slots of all live +requests share one device pool, indexed by the slot table of block rounds. A step +gathers the slots of its requests, updates them in one batch and scatters them +back. Thus its number of launched operations does not grow with the number of +requests. Inactive slots have masked KV writes and zero attention lengths. Fixed-shape prelude/coda calls can include unused rows; this avoids a host dependency for selecting dynamic batch sizes. Pending steps cannot be preempted. Waiting admissions drain outstanding steps so @@ -1074,6 +1078,22 @@ The saved work did not offset selection and result-copy overhead. The experiment was reverted; the runtime retains the simpler fixed-slot boundary calls. These results do not rule out a different implementation or workload benefiting. +### Boundary-KV models + +Dory's PRELUDE and CODA contain attention and keep their KV at depth 0. Its +asynchronous rounds keep the fixed-slot boundary calls above. Each call gets +depth-0 device metadata, and a pipeline step adds a write mask: + +- PRELUDE stores KV only for rows at depth 0. A retained row keeps the KV of + its first pass. +- CODA stores KV only for rows that reach the draft or the target depth. A row + between them keeps the KV of its draft pass, which later draft rows attend. + +With CUDA Graphs, `DeviceBoundaryGraphs` replays both stages in the same buckets +as the recurrent loops, and the write mask is a graph input. Asynchronous Dory +rounds take greedy requests only. `docs/dory.md` describes the KV contract and +its validation. + ## Validation The following commands and historical evidence concern the cross-round block diff --git a/docs/dory.md b/docs/dory.md index 46cfd8d6..b0fda856 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -86,6 +86,9 @@ layer id rather than their physical sublayer index. the next input stage and only its readback is deferred. The two switches are independent: sync eager, sync Graph, async eager and async Graph are tested separately. +- Self-speculation in block and token-pipeline mode, in the same four execution + modes. Asynchronous rounds take greedy requests only. See + [Self-speculation](#self-speculation). - The official tokenizer, including the Hub's `tokenizer.json` and chat template, resolved without requiring transformers to accept the checkpoint config (its generic validator rejects Dory's `mlp` sublayer entries). @@ -136,9 +139,9 @@ These fail fast rather than degrade silently: - WCPB, mixed or staged prefill, and PD. - Prefix caching. -- Self-speculation combined with CUDA Graphs, asynchronous scheduling, - preemption or `last_loop` KV, and sampled token-pipeline speculation. The - synchronous eager form is supported; see below. +- Self-speculation combined with preemption or `last_loop` KV, sampled + token-pipeline speculation, and sampled asynchronous rounds. The other forms + are supported; see below. - Per-request loop counts other than the configured full depth: Dory has no trained exit gate, so the runtime refuses to invent one, including via `min_loops` with `exit_threshold=0`. @@ -155,13 +158,15 @@ construction instead of attending full history. Masking accepts host-planned, graph-private and device-owned metadata; storage reclamation of out-of-window pages stays with 2B. -## Synchronous self-speculation +## Self-speculation `SpeculativeConfig(K, draft_loops=d, target_loops=T, mode=...)` drafts with the first `d` recurrent loops and verifies with all `T`, where `T` is the loop count -the model was loaded with. It needs `per_loop` KV, synchronous eager execution -and fixed-depth requests (`min_loops == max_loops == T`). Qualified depth pairs -are 1→2, 1→4 and 2→4. +the model was loaded with. It needs `per_loop` KV and fixed-depth requests +(`min_loops == max_loops == T`). Qualified depth pairs are 1→2, 1→4 and 2→4. +This section describes synchronous eager rounds first; +[asynchronous rounds and CUDA Graphs](#asynchronous-rounds-and-cuda-graphs) keep +the same KV contract. | Mode | Sampling | Round | | --- | --- | --- | @@ -222,6 +227,49 @@ final consumer completes. The pipeline mode uses the same preparation-local layout store and operation indices for its new-token, recurrent and endpoint views; all views are bound before execution. +### Asynchronous rounds and CUDA Graphs + +`ExecutionConfig(async_scheduling=True)` submits the next round before the host +reads the previous one (see `docs/async-speculative.md`). The actual positions +then exist only on the device. Thus the runner builds device metadata +(`DeviceKVMetadata`) at depth 0 for PRELUDE and CODA, as it does for the +recurrent loops. The KV contract does not change: all KV below the start of the +next round is the serial target history. + +- **Block rounds.** Each draft offset gives its rows one depth-0 view for + PRELUDE and the draft CODA. A partial draft CODA gets its own view of the + drafting rows. Verification gives every row one depth-0 view for the target + CODA. +- **Pipeline steps.** Every row runs both boundary stages, so each call keeps one + shape (the fixed-slot calls of `docs/async-speculative.md`). Write masks select + the rows that store KV. PRELUDE stores only for rows at depth 0. CODA stores + only for rows that reach the draft or the target depth. A row between these + depths (1→4) keeps the CODA KV of its draft pass, as in a synchronous step. + Both stages use one depth-0 view and differ only in the write mask, as in + synchronous steps and asynchronous block rounds. + The runtime discards the PRELUDE output of rows that are not at depth 0, and + the CODA output of rows that reach neither depth. +- **Sampling.** Asynchronous rounds take greedy requests only. Asynchronous + block rounds can sample for models without boundary KV, but that combination + is not qualified for Dory. Such a request fails when it is added. + +The asynchronous pipeline keeps all its slots full: when the root reaches the +target depth, the newest draft row gets its successor in the same step. The +synchronous pipeline appends only inside the scheduled span. The two modes +therefore run different step sequences, and their acceptance can differ. Both +commit the target tokens. + +`ExecutionConfig(cuda_graphs=True)`: + +- **Asynchronous rounds** replay bucketed graphs for PRELUDE, every recurrent + loop and CODA (`DeviceBoundaryGraphs`, `DeviceRecurrentGraphs`). Positions, + write addresses, write masks and context lengths are graph inputs. The engine + captures all buckets at construction. A row count above the largest bucket + runs eager. +- **Synchronous rounds** use the host-planned boundary graphs of ordinary decode + (`PreludeGraphs`, and `CodaGraphs` with the cache). These capture one graph + per row count, up to `cuda_graph_max_graphs`; other row counts run eager. + ## Validation status `tests/test_dory.py` and `tests/test_dory_gpu.py` hold the regression suites; @@ -390,6 +438,165 @@ Reading the numbers: memory was 74–82 GiB with the automatically sized KV pool; every arm released its KV blocks. +### Asynchronous self-speculation and CUDA Graphs + +`tests/test_dory_speculative_gpu.py` (97 CUDA cases, small FP32 model, Triton) +checks the execution modes against the dense oracle: + +- Greedy tokens for ragged requests in block and pipeline mode, for 1→2, 1→4 and + 2→4 and K = 1 and 3, in async eager, async Graph and sync Graph. +- A forced rejection at every draft position of an async block round, and forced + acceptance patterns of an async pipeline. After every collected round, every + plane below the committed frontier equals the serial target history while the + next round is in flight. +- After every async pipeline step, a retained row keeps its PRELUDE KV and a row + between the draft and target depths keeps the CODA KV of its draft pass, bit + for bit. +- No host read of a device value during submission, cancellation with two + rounds in flight and ID reuse, cleanup after a failed round, and the greedy + and preemption rejections. + +The Graph cases also check that all three graph families replay; async Graph +rounds have no eager fallback. On one H100 (torch 2.15.0+cu132, Triton 3.9), the +file passes together with `tests/test_dory_speculative.py`, +`tests/test_dory_gpu.py`, `tests/test_speculative.py`, `tests/test_cuda_graph.py` +and the `tests/test_async_*` files. The `flash_attn` cases were not selected, +because that package is not installed there. + +Real checkpoint (pinned revision), K = 3, greedy, Triton, one H100, torch +2.15.0+cu132. Prompts are disjoint windows of the WikiText-103 test split. + +**FP32 token agreement.** 32 requests × 512 prompt tokens + 128 generated +tokens. Every speculative arm (sync and async eager pipeline and block, sync +Graph pipeline, async Graph pipeline and block) is compared with synchronous +eager decoding without speculation at the same loop count: + +| Loops | Draft → target | Requests with identical tokens, every arm | Acceptance, pipeline | Acceptance, block | +| --- | --- | --- | --- | --- | +| 2 | 1 → 2 | 32/32 | 0.85 | 0.85 | +| 4 | 1 → 4 | 32/32 | 0.81 sync, 0.73 async | 0.85 | +| 4 | 2 → 4 | 32/32 | 0.93 | 0.92 | + +At 1 → 4 the asynchronous pipeline runs a different step sequence (see +[Asynchronous rounds and CUDA Graphs](#asynchronous-rounds-and-cuda-graphs)), +so its acceptance differs. Its tokens do not. + +**GSM8K, BF16.** The first 200 test questions, 8-shot prompt from the first +eight training examples, at most 256 generated tokens. An answer is correct +when the number after `####` equals the reference: + +| Arm | Correct | +| --- | --- | +| Baseline: async Graph, no speculation | 161/200 | +| Async Graph pipeline | 164/200 | +| Async Graph block | 163/200 | + +BF16 tokens depend on the batch composition, so most sequences differ from the +baseline after some token. Accuracy does not drop. + +**BF16 wall time.** Two loops, a one-loop draft. All requests are sent at the +same time. Each arm runs one warmup, then five timed repetitions, all in one +container. Wall times are the mean whole-run time in seconds, prefill included. +Every Graph arm uses `cuda_graph_max_graphs=128`, so no arm runs eager because +its graph cache is full; every Graph arm had zero graph fallbacks. BF16 +acceptance is 0.85–0.88. Tables 1–4 were measured before PRELUDE and CODA of a +pipeline step shared one depth-0 view. + +Table 1: with CUDA Graphs, concurrency sweep. Prompt 1024 tokens, 128 generated +tokens for each request. Percentages are against the baseline. + +| Requests | Baseline (async Graph, no spec) | Baseline, default graph cache | Sync Graph pipeline | Async Graph pipeline | Async Graph block | +| --- | --- | --- | --- | --- | --- | +| 1 | 2.50 | 2.49 (−0.1 %) | 1.83 (−27 %) | **1.78 (−29 %)** | 1.91 (−23 %) | +| 4 | 3.17 | 3.18 (+0.4 %) | 2.66 (−16 %) | **2.63 (−17 %)** | 2.95 (−7 %) | +| 16 | 5.89 | 5.90 (+0.1 %) | 5.51 (−6 %) | **5.30 (−10 %)** | 5.63 (−5 %) | +| 64 | 15.60 | 25.72 (+65 %) | 16.00 (+3 %) | **15.10 (−3 %)** | 16.45 (+5 %) | + +Table 2: latency with CUDA Graphs, same workload. TTFT is the time to the first +token; TPOT is the time per output token after the first token. Percentages are +against the baseline at the same concurrency. + +| Requests | Arm | TTFT p50 (s) | TTFT p90 (s) | TPOT p50 (ms) | TPOT p90 (ms) | +| --- | --- | --- | --- | --- | --- | +| 1 | Baseline (async Graph, no spec) | 0.18 | 0.18 | 18.3 | 18.3 | +| 1 | Async Graph pipeline | 0.17 (−2 %) | 0.20 (+13 %) | 12.6 (−31 %) | 12.6 (−31 %) | +| 1 | Async Graph block | 0.17 (−1 %) | 0.19 (+7 %) | 13.7 (−25 %) | 13.7 (−25 %) | +| 4 | Baseline (async Graph, no spec) | 0.67 | 0.67 | 20.0 | 20.0 | +| 4 | Async Graph pipeline | 0.66 (−1 %) | 0.67 (−0.8 %) | 16.6 (−17 %) | 17.4 (−13 %) | +| 4 | Async Graph block | 0.69 (+3 %) | 0.69 (+3 %) | 18.1 (−10 %) | 19.0 (−5 %) | +| 16 | Baseline (async Graph, no spec) | 2.01 | 2.75 | 35.2 | 40.9 | +| 16 | Async Graph pipeline | 1.68 (−17 %) | 2.70 (−2 %) | 28.8 (−18 %) | 36.1 (−12 %) | +| 16 | Async Graph block | 1.79 (−11 %) | 2.93 (+6 %) | 30.1 (−15 %) | 36.1 (−12 %) | +| 64 | Baseline (async Graph, no spec) | 6.21 | 10.58 | 76.6 | 108.5 | +| 64 | Async Graph pipeline | 5.78 (−7 %) | 10.02 (−5 %) | 70.9 (−7 %) | 102.5 (−5 %) | +| 64 | Async Graph block | 6.43 (+3 %) | 11.54 (+9 %) | 68.1 (−11 %) | 96.4 (−11 %) | + +Table 3: without CUDA Graphs (eager), sync against async, same workload. There +is no eager baseline without speculation, so each async percentage is against +the sync value of the same mode. + +| Requests | Pipeline sync | Pipeline async | Block sync | Block async | +| --- | --- | --- | --- | --- | +| 1 | 6.80 | 6.42 (−6 %) | 6.83 | 7.16 (+5 %) | +| 4 | 7.80 | 8.24 (+6 %) | 9.64 | 9.71 (+0.8 %) | +| 16 | 10.74 | 10.57 (−2 %) | 13.24 | 13.25 (+0.1 %) | +| 64 | 21.22 | 20.49 (−3 %) | 28.02 | 27.29 (−3 %) | + +Table 4: long context. 8 requests, prompt 8192 tokens, 128 generated tokens for +each request. + +| Arm | Wall (s) | TTFT p50 (s) | TTFT p90 (s) | TPOT p50 (ms) | TPOT p90 (ms) | +| --- | --- | --- | --- | --- | --- | +| Baseline (async Graph, no spec) | 23.73 | 18.60 | 20.71 | 53.2 | 68.9 | +| Sync Graph pipeline | 23.52 (−0.9 %) | 17.85 (−4 %) | 20.73 (+0.1 %) | 51.2 (−4 %) | 72.5 (+5 %) | +| Async Graph pipeline | 23.18 (−2 %) | 17.69 (−5 %) | 20.58 (−0.6 %) | 49.4 (−7 %) | 70.3 (+2 %) | +| Async Graph block | 23.13 (−3 %) | 17.60 (−5 %) | 20.42 (−1 %) | 49.9 (−6 %) | 67.9 (−1 %) | +| Sync eager pipeline | 28.83 | 18.04 | 21.08 | 86.1 | 111.4 | +| Async eager pipeline | 28.52 (−1 % vs sync) | 18.10 (+0.3 % vs sync) | 21.15 (+0.3 % vs sync) | 87.2 (+1 % vs sync) | 108.0 (−3 % vs sync) | + +Reading the numbers: + +- The gain of speculation falls as concurrency rises. At one request the GPU + has spare capacity, so draft and verify work is almost free. At 64 requests + the GPU is busy, and the extra verify rows cancel most of the saved steps. +- In pipeline mode, every row runs both boundary stages, so that each call keeps + one shape. PRELUDE also runs on rows that already hold their input-stage KV, + and CODA also runs on rows that reach neither the draft nor the target depth. + The runtime discards these outputs. At 64 requests, this extra work can + explain a part of the small gain. +- At 8 × 8192, prefill takes about 18 of 24 seconds. Speculation shortens only + decode, so the whole run is 2–3 % faster. +- The baseline with the default graph cache (16 shapes) runs eager in most calls + at 64 requests and takes 25.72 s instead of 15.60 s. Comparisons against that + baseline overstate the gain of speculation. +- These runs use a different harness and host than the 2D and 3A tables above, + so the absolute numbers do not compare with those tables. +- Not covered: HTTP serving and open-loop arrival traces. + +**Known issues.** + +- Async Graph block is 5 % slower than the baseline at 64 requests. Its time per + output token is 11 % lower, but its time to the first token is 3–9 % higher. + The async Graph pipeline is 3 % faster there, so it is the better mode at high + concurrency. +- Without CUDA Graphs, async rounds at 1 and 4 requests are between 6 % faster + and 6 % slower than sync rounds (Table 3). A dedicated run alternated a sync + and an async engine in ABBA order, 10 timed repetitions each, at one request + in pipeline mode. Async was 2.7 % and 3.9 % slower in two containers. The + profiler counted 5503 kernel launches per generated token for sync and 5626 + for async (+124, +2.2 %). Async rounds keep positions on the GPU, so the GPU + computes the KV metadata, the PRELUDE and CODA write masks and the slot-pool + update; sync rounds do this work on the CPU. Eager Dory is bound by host-side + launch cost, so the extra launches give the extra time. At 16 and 64 requests, + async eager is 0–3 % faster than sync eager. +- After PRELUDE and CODA of a pipeline step shared one depth-0 view, async makes + 5615 launches per generated token (+113 against sync). The same ABBA run gave + −0.2 % and +0.4 % in two new containers, and async was slower in 5 of 10 + rounds in each. The time between containers varies by a few percent, so these + runs do not show that the gap is closed. A fused kernel for the per-step + update and metadata would remove more launches. It also changes the Ouro path, + so it is a separate change. + ### Not yet performed - A runtime-to-runtime comparison against vLLM once a Dory-capable build is @@ -402,7 +609,7 @@ Reading the numbers: - Graph-captured prefill, open-loop arrival traces and sampled decoding on the real checkpoint in Graph or async mode. The measurements above are closed-loop, greedy, and their time to first token is eager prefill. -- Self-speculation with CUDA graphs, asynchronous scheduling, preemption, - sampled token-pipeline, adaptive exit, or a draft width above the fixed K. +- Self-speculation with preemption, sampled asynchronous rounds, sampled + token-pipeline, adaptive exit, or a draft width above the fixed K. - Sampled speculation on the real checkpoint (the rejection-sampling contract is qualified on the tiny CPU model only). diff --git a/tests/test_async_speculative.py b/tests/test_async_speculative.py index e777e967..2b630a54 100644 --- a/tests/test_async_speculative.py +++ b/tests/test_async_speculative.py @@ -132,7 +132,7 @@ def test_second_round_executes_before_cpu_commit_at_every_rejection(accepted, mo start = request.position calls = 0 - def controlled(hidden): + def controlled(hidden, boundary=None): nonlocal calls phase = calls % 4 calls += 1 @@ -182,7 +182,7 @@ def test_eos_discards_queued_round_and_request_id_can_be_reused(monkeypatch, gra monkeypatch.setattr( e.speculative_runner, "_coda", - lambda h: torch.zeros( + lambda h, boundary=None: torch.zeros( len(h), model.config.vocab_size, device="cuda", diff --git a/tests/test_async_speculative_graphs.py b/tests/test_async_speculative_graphs.py index 04ef90e5..67adb2f0 100644 --- a/tests/test_async_speculative_graphs.py +++ b/tests/test_async_speculative_graphs.py @@ -99,7 +99,7 @@ def test_two_graph_rounds_keep_device_frontier_and_rejection_mask(backend, accep written_before = [[layer.prefix for layer in depth] for depth in allocation.written] calls = 0 - def controlled(hidden): + def controlled(hidden, boundary=None): nonlocal calls phase = calls % 4 calls += 1 @@ -165,7 +165,9 @@ def test_graph_eos_retires_later_round_before_id_reuse(backend, monkeypatch): patch.setattr( runner, "_coda", - lambda h: torch.zeros(len(h), engine.model.config.vocab_size, device=h.device), + lambda h, boundary=None: torch.zeros( + len(h), engine.model.config.vocab_size, device=h.device + ), ) engine.step() engine.step() diff --git a/tests/test_async_speculative_lifecycle.py b/tests/test_async_speculative_lifecycle.py index bba1076b..9b1c24b9 100644 --- a/tests/test_async_speculative_lifecycle.py +++ b/tests/test_async_speculative_lifecycle.py @@ -164,7 +164,9 @@ def test_cuda_serving_two_completed_rounds_isolate_decode_failure( monkeypatch.setattr( engine.speculative_runner, "_coda", - lambda h: torch.zeros(len(h), engine.model.config.vocab_size, device=h.device), + lambda h, boundary=None: torch.zeros( + len(h), engine.model.config.vocab_size, device=h.device + ), ) engine.step() engine.step() diff --git a/tests/test_async_speculative_sampling_plan.py b/tests/test_async_speculative_sampling_plan.py index 002aa439..882d3eda 100644 --- a/tests/test_async_speculative_sampling_plan.py +++ b/tests/test_async_speculative_sampling_plan.py @@ -207,15 +207,18 @@ def __init__(self, monkeypatch, seed): num_speculative_tokens=self.K, draft_loops=1, target_loops=2 ) runner.device = torch.device("cpu") + runner.boundary_kv = False runner.rng = RngRegistry() runner.slot_values = torch.zeros((8, 2), dtype=torch.long) runner.runtime = SimpleNamespace( - forward=lambda batch: SimpleNamespace(hidden=self.E[batch.tokens]) + prepare=lambda batch: batch, + forward=lambda batch: SimpleNamespace(hidden=self.E[batch.tokens]), ) + runner.prelude_graphs = None runner._execute_recurrent_step = lambda hidden, positions, tables, depth, allocations: ( hidden if depth < 1 else hidden @ self.M ) - runner._coda = lambda hidden: hidden @ self.W + runner._coda = lambda hidden, boundary=None: hidden @ self.W self.runner = runner def run(self, specs): diff --git a/tests/test_dory_speculative.py b/tests/test_dory_speculative.py index e52b2832..c214cfb4 100644 --- a/tests/test_dory_speculative.py +++ b/tests/test_dory_speculative.py @@ -426,11 +426,11 @@ def distribution(tokens): def test_unqualified_speculative_combinations_fail_at_construction(): model = dory(2) - with pytest.raises(ValueError, match="synchronous"): + with pytest.raises(ValueError, match="requires CUDA"): engine(model, 2, 1, execution_config=ExecutionConfig(async_scheduling=True)) with pytest.raises(ValueError, match="CUDA graphs"): engine(model, 2, 1, execution_config=ExecutionConfig(cuda_graphs=True)) - with pytest.raises(ValueError, match="without CUDA graphs or preemption"): + with pytest.raises(ValueError, match="does not support preemption"): engine( model, 2, diff --git a/tests/test_dory_speculative_gpu.py b/tests/test_dory_speculative_gpu.py new file mode 100644 index 00000000..e3df0641 --- /dev/null +++ b/tests/test_dory_speculative_gpu.py @@ -0,0 +1,447 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Dory self-speculation on CUDA: asynchronous rounds and CUDA Graphs. + +Every case needs an explicit ``CUDA_VISIBLE_DEVICES`` and ``--run-gpu``. Each +execution mode must serve the greedy tokens of the dense oracle. Forced drafts +check that the committed KV of every plane is the serial target history while +the next round is already submitted. +""" + +import pytest +import torch + +from tests.reference.dory import dense_dory_reference +from tests.test_dory import make_cache, tiny_dory_config +from vllm_rlt import ( + CacheConfig, + ExecutionConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.models import DoryForCausalLM + +pytestmark = pytest.mark.gpu + +DEPTH_PAIRS = [(1, 2), (1, 4), (2, 4)] +# Prompts shorter than and longer than the tiny window of four. +PROMPTS = {"a": [4, 7, 3], "b": [9], "c": [5, 6, 8, 2, 11]} +LIMITS = {"a": 12, "b": 1, "c": 7} +EXECUTION = { + "async_eager": dict(asynchronous=True, graphs=False), + "async_graph": dict(asynchronous=True, graphs=True), + "sync_graph": dict(asynchronous=False, graphs=True), +} + + +def reference_pair(loops, **overrides): + """A CPU oracle model and a CUDA model with the same FP32 weights.""" + torch.manual_seed(42) + reference = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops, **overrides)).eval() + model = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops, **overrides)) + model.load_state_dict(reference.state_dict()) + return reference, model.to("cuda").eval() + + +def engine(model, k, draft, mode="block", *, asynchronous=True, graphs=False): + return LLMEngine( + model, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=1024, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=4), + execution_config=ExecutionConfig(async_scheduling=asynchronous, cuda_graphs=graphs), + speculative_config=SpeculativeConfig( + k, draft_loops=draft, target_loops=model.config.total_ut_steps, mode=mode + ), + ) + + +def params(model, max_tokens, **kwargs): + loops = model.config.total_ut_steps + kwargs.setdefault("ignore_eos", True) + return SamplingParams(max_tokens=max_tokens, min_loops=loops, max_loops=loops, **kwargs) + + +def drain(e): + final = {} + for _ in range(4000): + if not e.has_unfinished_requests(): + return final + for output in e.step(): + if output.finished: + final[output.request_id] = output + pytest.fail("speculative scheduler did not drain") + + +def greedy_reference(model, prompt, count): + tokens = list(prompt) + for _ in range(count): + logits = dense_dory_reference(model, torch.tensor(tokens))["logits"] + tokens.append(int(logits[-1].argmax())) + return tokens + + +def assert_released(e): + runner = e.speculative_runner + assert not runner.states + assert all(not buffer.leased for buffer in runner.buffers) + assert e.cache_manager.num_used_blocks == 0 + + +def assert_graphs_replayed(e): + runner = e.speculative_runner + for graphs in (runner.prelude_graphs, runner.graphs, runner.coda_graphs): + assert graphs.replays > 0 + + +def serial_cache(reference, tokens): + """KV of the reference schedule: one token per step at the full depth.""" + cache = make_cache(reference) + tokens = torch.tensor(tokens) + assert cache.allocate("request", len(tokens)) + for position in range(len(tokens)): + boundary = cache._prepare_batch(["request"], [0], [position]) + hidden = reference.prelude_prepared(tokens[position : position + 1], boundary, cache) + for depth in range(reference.config.total_ut_steps): + core = cache._prepare_batch(["request"], [depth], [position]) + hidden, _ = reference.recurrent_prepared(hidden, core, cache) + reference.coda_prepared(hidden, boundary, cache) + return cache + + +def device_kv(cache, layer, request_id, depth, length): + """Read KV that device-owned rounds wrote; the host written frontier lags it.""" + table = torch.tensor(cache.get_block_table(request_id, depth), dtype=torch.long) + positions = torch.arange(length) + blocks = table[positions // cache.block_size].to(cache.device) + offsets = (positions % cache.block_size).to(cache.device) + return cache.key_cache[blocks, layer, offsets], cache.value_cache[blocks, layer, offsets] + + +def assert_committed_kv_is_the_target_history(e, reference, request_id, tokens, frontier): + """Every plane below ``frontier`` holds what serial execution stores.""" + e.speculative_runner.synchronize() + expected = serial_cache(reference, tokens[:frontier]) + boundary = (*reference.prelude_kv_layers, *reference.coda_kv_layers) + for layer in range(reference.config.num_kv_layers): + depths = (0,) if layer in boundary else range(reference.config.total_ut_steps) + for depth in depths: + actual = device_kv(e.cache_manager, layer, request_id, depth, frontier) + wanted = expected.read(layer, "request", depth, frontier) + for a, b in zip(actual, wanted): + torch.testing.assert_close(a.cpu(), b, atol=2e-5, rtol=2e-4) + + +def force_drafts(monkeypatch, e, sequence, wrong): + """Make every draft row propose the reference token, or miss where ``wrong`` says. + + ``wrong(position, index)`` receives the input position of the draft row and, + in block mode, its draft offset in the round. Target rows keep the model logits. + The position read is a host synchronization that only this test adds. + """ + runner = e.speculative_runner + coda, draft = runner._coda, runner._draft + vocab = e.model.config.vocab_size + state = {"offset": None} + + def drafting(*args): + state["offset"] = 0 + try: + return draft(*args) + finally: + state["offset"] = None + + def controlled(hidden, boundary=None): + logits = coda(hidden, boundary) + positions = boundary.position_ids.tolist() + if runner.config.mode == "block": + if state["offset"] is None: + return logits + rows = [(row, state["offset"]) for row in range(len(positions))] + state["offset"] += 1 + else: + pipeline = runner.states["r"].pipeline + width = len(positions) + # The depths still describe the rows before this step. + depths = pipeline.depths[:width].tolist() + active = pipeline.active[:width].tolist() + rows = [(row, None) for row in range(width) if active[row] and depths[row] == 0] + for row, index in rows: + position = positions[row] + if position + 1 >= len(sequence): + continue + token = sequence[position + 1] + if wrong(position, index): + token = (token + 1) % vocab + logits[row] = -100.0 + logits[row, token] = 100.0 + return logits + + monkeypatch.setattr(runner, "_draft", drafting) + monkeypatch.setattr(runner, "_coda", controlled) + + +@pytest.mark.parametrize("execution", EXECUTION) +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("k", [1, 3]) +def test_greedy_matches_the_dense_reference(execution, mode, draft, target, k): + reference, model = reference_pair(target) + e = engine(model, k, draft, mode, **EXECUTION[execution]) + for rid, prompt in PROMPTS.items(): + e.add_request(rid, prompt, params(model, LIMITS[rid])) + final = drain(e) + for rid, prompt in PROMPTS.items(): + expected = greedy_reference(reference, prompt, LIMITS[rid])[len(prompt) :] + assert final[rid].token_ids == expected + assert final[rid].exit_depths == [target] * LIMITS[rid] + runner = e.speculative_runner + if EXECUTION[execution]["graphs"]: + assert_graphs_replayed(e) + if not EXECUTION[execution]["asynchronous"]: + assert not runner.pipeline_states + assert e.cache_manager.num_used_blocks == 0 + return + assert runner.submitted_before_collect > 0 + if EXECUTION[execution]["graphs"]: + # Every async row count fits a captured bucket. + for graphs in (runner.prelude_graphs, runner.graphs, runner.coda_graphs): + assert graphs.fallbacks == 0 + assert_released(e) + + +def bootstrap(e, model, prompt, count): + e.add_request("r", prompt, params(model, count)) + request = e.scheduler.requests["r"] + for _ in range(30): + if e.step(): + return request + pytest.fail("bootstrap did not produce a token") + + +@pytest.mark.parametrize("graphs", [False, True]) +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_submission_reads_no_device_value(mode, graphs, monkeypatch): + _, model = reference_pair(4) + e = engine(model, 3, 1, mode, graphs=graphs) + request = bootstrap(e, model, [4, 7, 3], 16) + + def forbidden(*args, **kwargs): + raise AssertionError("host read in cross-round GPU submission") + + with monkeypatch.context() as patch: + patch.setattr(torch.Tensor, "item", forbidden) + patch.setattr(torch.Tensor, "tolist", forbidden) + e.step() + e.step() + assert len(e._pending_speculative) == 2 + assert len(request.generated_token_ids) == 1 + e.abort_request("r") + drain(e) + assert_released(e) + + +@pytest.mark.parametrize("graphs", [False, True]) +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("rejected", [0, 1, 2, 3]) +def test_block_commits_target_history_through_each_rejection( + draft, target, rejected, graphs, monkeypatch +): + """rejected=3 accepts all three candidates and commits the bonus token.""" + reference, model = reference_pair(target) + prompt, count = [4, 7, 3], 13 + sequence = greedy_reference(reference, prompt, count) + e = engine(model, 3, draft, graphs=graphs) + force_drafts(monkeypatch, e, sequence, lambda position, index: index == rejected) + e.add_request("r", prompt, params(model, count)) + request = e.scheduler.requests["r"] + overlapped = False + while e.has_unfinished_requests(): + outputs = e.step() + overlapped |= len(e._pending_speculative) == 2 + generated = list(request.generated_token_ids) + assert generated == sequence[len(prompt) : len(prompt) + len(generated)] + if outputs and outputs[0].finished: + break + if generated and e.cache_manager.has_allocation("r"): + frontier = len(prompt) + len(generated) - 1 + assert_committed_kv_is_the_target_history(e, reference, "r", sequence, frontier) + drain(e) + assert overlapped + stats = e.speculative_runner.stats + if rejected: + assert stats.accepted_tokens > 0 + else: + assert stats.accepted_tokens == 0 + assert request.generated_token_ids == sequence[len(prompt) :] + if graphs: + assert_graphs_replayed(e) + assert_released(e) + + +@pytest.mark.parametrize("graphs", [False, True]) +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("pattern", ["all", "none", "every_third"]) +def test_pipeline_commits_target_history_under_forced_acceptance( + draft, target, pattern, graphs, monkeypatch +): + reference, model = reference_pair(target) + prompt, count = [4, 7, 3], 12 + sequence = greedy_reference(reference, prompt, count) + wrong = { + "all": lambda position, index: False, + "none": lambda position, index: True, + "every_third": lambda position, index: position % 3 == 0, + }[pattern] + e = engine(model, 3, draft, "pipeline", graphs=graphs) + force_drafts(monkeypatch, e, sequence, wrong) + e.add_request("r", prompt, params(model, count)) + request = e.scheduler.requests["r"] + while e.has_unfinished_requests(): + outputs = e.step() + generated = list(request.generated_token_ids) + assert generated == sequence[len(prompt) : len(prompt) + len(generated)] + if outputs and outputs[0].finished: + break + if generated and e.cache_manager.has_allocation("r"): + frontier = len(prompt) + len(generated) - 1 + assert_committed_kv_is_the_target_history(e, reference, "r", sequence, frontier) + drain(e) + stats = e.speculative_runner.stats + verified = count - 1 + assert stats.committed_tokens == verified + if pattern == "all": + assert stats.accepted_tokens == verified + elif pattern == "none": + assert stats.accepted_tokens == 0 + else: + assert 0 < stats.accepted_tokens < verified + if graphs: + assert_graphs_replayed(e) + assert_released(e) + + +def stage_kv(e, layers, position): + """KV of one position in every layer of one boundary stage.""" + return torch.stack( + [ + torch.stack(device_kv(e.cache_manager, layer, "r", 0, position + 1))[:, position] + for layer in layers + ] + ) + + +@pytest.mark.parametrize("graphs", [False, True]) +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +def test_pipeline_keeps_the_boundary_kv_of_rows_in_flight(draft, target, graphs, monkeypatch): + """A pipeline step stores boundary KV only for the rows that use that stage. + + A retained row keeps the input-stage KV of its first pass. A row between + the draft and target depths keeps the output-stage KV of its draft pass, + which the draft rows behind it attend. + """ + _, model = reference_pair(target, ngpt_alpha_init=1.0) + e = engine(model, 3, draft, "pipeline", graphs=graphs) + runner = e.speculative_runner + coda = runner._coda + steps = [] + + def recording(hidden, boundary=None): + # Inside the step, the device depths still describe the rows before it. + pipeline = runner.states["r"].pipeline + positions = boundary.position_ids.tolist() + depths = pipeline.depths[: len(positions)].tolist() + active = pipeline.active[: len(positions)].tolist() + steps.append([(p, d) for p, d, a in zip(positions, depths, active) if a]) + return coda(hidden, boundary) + + monkeypatch.setattr(runner, "_coda", recording) + e.add_request("r", [4, 7, 3], params(model, 24)) + first, drafted = {}, {} + retained = between = 0 + while e.has_unfinished_requests(): + submitted = len(steps) + e.step() + if len(steps) == submitted or not e.cache_manager.has_allocation("r"): + continue + runner.synchronize() + for position, depth in steps[-1]: + prelude = stage_kv(e, model.prelude_kv_layers, position) + output = stage_kv(e, model.coda_kv_layers, position) + if depth == 0: + first[position], drafted[position] = prelude, output + continue + assert torch.equal(prelude, first[position]) + retained += 1 + if depth + draft < target: + assert torch.equal(output, drafted[position]) + between += 1 + assert retained + assert bool(between) == (target - draft > draft) + if graphs: + assert_graphs_replayed(e) + assert_released(e) + + +@pytest.mark.parametrize("graphs", [False, True]) +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_cancellation_with_rounds_in_flight_allows_id_reuse(mode, graphs): + reference, model = reference_pair(4) + e = engine(model, 3, 2, mode, graphs=graphs) + e.add_request("a", [4, 7, 3], params(model, 14)) + e.add_request("b", [9, 5], params(model, 8)) + final = {} + for _ in range(30): + final.update({output.request_id: output for output in e.step() if output.finished}) + if len(e._pending_speculative) == 2: + break + assert len(e._pending_speculative) == 2 and "a" not in final + assert e.abort_request("a").finish_reason == "abort" + e.add_request("a", [6, 2], params(model, 5)) + final.update(drain(e)) + assert final["a"].token_ids == greedy_reference(reference, [6, 2], 5)[2:] + assert final["b"].token_ids == greedy_reference(reference, [9, 5], 8)[2:] + assert_released(e) + + +@pytest.mark.parametrize("graphs", [False, True]) +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_failed_round_releases_requests_and_allows_id_reuse(mode, graphs, monkeypatch): + reference, model = reference_pair(2) + e = engine(model, 3, 1, mode, graphs=graphs) + bootstrap(e, model, [4, 7, 3], 8) + runner = e.speculative_runner + name = "_pipeline_step" if mode == "pipeline" else "_verify" + original = getattr(runner, name) + + def fail(*args): + original(*args) + raise RuntimeError("injected failure after submission") + + monkeypatch.setattr(runner, name, fail) + with pytest.raises(RuntimeError, match="injected failure"): + drain(e) + assert not e._pending_speculative and not e._retiring_speculative + assert not e.scheduler.requests + assert_released(e) + monkeypatch.setattr(runner, name, original) + e.add_request("r", [5], params(model, 3)) + assert drain(e)["r"].token_ids == greedy_reference(reference, [5], 3)[1:] + assert_released(e) + + +def test_unqualified_async_combinations_are_rejected(): + _, model = reference_pair(2) + e = engine(model, 2, 1) + with pytest.raises(ValueError, match="greedy sampling"): + e.add_request("r", [4], params(model, 2, temperature=0.7)) + with pytest.raises(ValueError, match="preemption"): + LLMEngine( + model, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=4, enable_preemption=True), + execution_config=ExecutionConfig(async_scheduling=True), + speculative_config=SpeculativeConfig(2, draft_loops=1, target_loops=2), + ) diff --git a/tests/test_speculative.py b/tests/test_speculative.py index 7efb6992..a18cb79b 100644 --- a/tests/test_speculative.py +++ b/tests/test_speculative.py @@ -1207,7 +1207,7 @@ def test_async_pipeline_uses_gpu_decision_before_collect(reject, monkeypatch): runner = e.speculative_runner calls = 0 - def controlled(hidden): + def controlled(hidden, boundary=None): nonlocal calls calls += 1 logits = torch.full( diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index ce700908..696e72dd 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -99,17 +99,9 @@ def __init__( if speculative_config is not None: if not getattr(model, "supports_self_speculation", False): raise ValueError("boundary-KV speculative decoding is not yet supported") - # Qualified: synchronous eager rounds. Async rounds, Graph replay and - # resumable preemption of draft/target boundary state are separate work. - if ( - self.execution_config.async_scheduling - or self.execution_config.cuda_graphs - or scheduler_config.enable_preemption - ): - raise ValueError( - "boundary-KV speculative decoding requires synchronous eager " - "execution without CUDA graphs or preemption" - ) + # Resumable preemption of draft/target boundary state is separate work. + if scheduler_config.enable_preemption: + raise ValueError("boundary-KV speculative decoding does not support preemption") if not getattr(model, "supports_cuda_graphs", True) and self.execution_config.cuda_graphs: raise ValueError("this model does not support CUDA graphs yet") self.speculative_config = speculative_config @@ -362,10 +354,15 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i if not isinstance(request_id, str) or not request_id: raise ValueError("request_id must be a nonempty string") params = sampling_params or SamplingParams() - if ( - self.speculative_config and self.speculative_config.mode == "pipeline" - ) and params.temperature != 0: - raise ValueError("pipeline speculative decoding currently requires greedy sampling") + if params.temperature != 0: + if self.pipeline_speculative: + raise ValueError("pipeline speculative decoding currently requires greedy sampling") + # RFC #176 qualifies asynchronous boundary-KV rounds for greedy requests only. + if self.async_speculative and getattr(self.model, "requires_boundary_kv", False): + raise ValueError( + "asynchronous boundary-KV speculative decoding currently requires " + "greedy sampling" + ) if request_id in self._retiring_speculative: self._retire_speculative(wait=True, request_id=request_id) config = self.model.config diff --git a/vllm_rlt/models/dory.py b/vllm_rlt/models/dory.py index 8e6a388e..a26e1145 100644 --- a/vllm_rlt/models/dory.py +++ b/vllm_rlt/models/dory.py @@ -488,8 +488,8 @@ class DoryForCausalLM(nn.Module): supports_async_scheduling = True # Selection must reject a backend that would silently attend full history. requires_sliding_window = True - # Synchronous self-speculation with per-loop KV: a draft's loops are the - # target's first loops, and boundary KV is committed by the target pass. + # Self-speculation with per-loop KV, in every execution mode: a draft's loops + # are the target's first loops, and boundary KV is committed by the target pass. supports_self_speculation = True def __init__(self, config: DoryConfig) -> None: diff --git a/vllm_rlt/worker/async_speculative.py b/vllm_rlt/worker/async_speculative.py index 8b18dfa2..fac805a5 100644 --- a/vllm_rlt/worker/async_speculative.py +++ b/vllm_rlt/worker/async_speculative.py @@ -5,6 +5,11 @@ verify with speculative rejection sampling against ``p`` (see ``speculative_sampling``) and return their committed sequence (accepted drafts, then correction or bonus) in the result rows. A batch without random requests takes the greedy path with no extra device work. + +Boundary-KV models (attention in PRELUDE and CODA) keep the history of both stages at +depth 0. The contract is the same as in ``speculative``: all KV below the start of the +next round is the serial target history. The boundary stages use device metadata, as +the recurrent loops do, because the actual positions exist only on the device. """ import math @@ -20,6 +25,7 @@ from vllm_rlt.request import Operation, Request from vllm_rlt.worker.cuda_graph import ( CodaGraphs, + DeviceBoundaryGraphs, DeviceRecurrentGraphs, speculative_capture_sizes, ) @@ -200,7 +206,11 @@ def merge_random_round( @dataclass class DevicePipelineState: - """Fixed-capacity, oldest-first token slots; all validity and depths live on GPU.""" + """Fixed-capacity, oldest-first token slots; all validity and depths live on GPU. + + The runner keeps one pool with a leading slot dimension. A request state holds + the views of its own slot. + """ tokens: torch.Tensor depths: torch.Tensor @@ -208,6 +218,22 @@ class DevicePipelineState: candidates: torch.Tensor active: torch.Tensor + @classmethod + def zeros(cls, slots, width, hidden_size, dtype, device): + return cls( + torch.zeros((slots, width), dtype=torch.long, device=device), + torch.zeros((slots, width), dtype=torch.long, device=device), + torch.zeros((slots, width, hidden_size), dtype=dtype, device=device), + torch.zeros((slots, width), dtype=torch.long, device=device), + torch.zeros((slots, width), dtype=torch.bool, device=device), + ) + + def tensors(self): + return (self.tokens, self.depths, self.hidden, self.candidates, self.active) + + def row(self, slot): + return DevicePipelineState(*(tensor[slot] for tensor in self.tensors())) + @dataclass class SpeculativeRequestState: @@ -363,19 +389,17 @@ def __init__( compute_stream=compute_stream, ) sizes = speculative_capture_sizes(execution, scheduler, config) - self.graphs = ( - DeviceRecurrentGraphs(model, cache, execution, False, capture_sizes=sizes) - if execution.cuda_graphs - else None - ) - self.coda_graphs = ( - CodaGraphs(model, execution, bucketed=True, capture_sizes=sizes) - if execution.cuda_graphs - else None - ) - if self.graphs is not None: + if execution.cuda_graphs: + self.graphs = DeviceRecurrentGraphs(model, cache, execution, False, capture_sizes=sizes) + if self.boundary_kv: + self.prelude_graphs = DeviceBoundaryGraphs(model, cache, Operation.PRELUDE, sizes) + self.coda_graphs = DeviceBoundaryGraphs(model, cache, Operation.CODA, sizes) + else: + self.coda_graphs = CodaGraphs(model, execution, bucketed=True, capture_sizes=sizes) self.graphs.prepare() self.coda_graphs.prepare() + if self.prelude_graphs is not None: + self.prelude_graphs.prepare() self.compute_stream = compute_stream self.copy_stream = copy_stream self.compute_stream.wait_stream(torch.cuda.current_stream(self.device)) @@ -388,6 +412,22 @@ def __init__( ) self.free_slots = list(reversed(range(len(self.slot_values)))) self._retired_slot_values: list[torch.Tensor] = [] + # Pipeline token slots of every live request, indexed like the slot table, so + # a step gathers, updates and scatters all its requests in one operation each. + self.pipeline_width = min( + config.num_speculative_tokens + 1, config.target_loops // config.draft_loops + ) + self.pipeline_pool = ( + DevicePipelineState.zeros( + len(self.slot_values), + self.pipeline_width, + model.config.hidden_size, + next(model.parameters()).dtype, + self.device, + ) + if config.mode == "pipeline" + else None + ) self.submitted_before_collect = 0 self.discarded_rounds = 0 self.readback_waits = 0 @@ -404,36 +444,29 @@ def _state(self, request, buffer): host = torch.tensor([request.input_token_id, request.position], pin_memory=True) values = host.to(self.device, non_blocking=True) buffer.keepalive.extend((host, values)) - if self.config.mode == "pipeline": - state = SpeculativeRequestState(request, allocation, values[0], values[1]) - else: - if not self.free_slots: - self._grow_slots() - slot = self.free_slots.pop() - self.slot_values[slot].copy_(values) - state = SpeculativeRequestState( - request, allocation, self.slot_values[slot, 0], self.slot_values[slot, 1], slot=slot - ) - if self.config.mode == "pipeline": - width = min( - self.config.num_speculative_tokens + 1, - self.config.target_loops // self.config.draft_loops, - ) - parameter = next(self.model.parameters()) - state.pipeline = DevicePipelineState( - state.token.expand(width).clone(), - torch.zeros(width, dtype=torch.long, device=self.device), - torch.zeros( - (width, self.model.config.hidden_size), - dtype=parameter.dtype, - device=self.device, - ), - torch.zeros(width, dtype=torch.long, device=self.device), - torch.arange(width, device=self.device) == 0, - ) + if not self.free_slots: + self._grow_slots() + slot = self.free_slots.pop() + self.slot_values[slot].copy_(values) + if self.pipeline_pool is not None: + pool = self.pipeline_pool + pool.tokens[slot].copy_(values[0].expand(self.pipeline_width)) + for tensor in (pool.depths, pool.hidden, pool.candidates, pool.active): + tensor[slot].zero_() + pool.active[slot, 0] = True + state = SpeculativeRequestState( + request, allocation, *self.slot_values[slot].unbind(), slot=slot + ) + if self.pipeline_pool is not None: + state.pipeline = self.pipeline_pool.row(slot) self.states[rid] = state return state + def bind_slot(self, state): + state.token, state.position = self.slot_values[state.slot].unbind() + if self.pipeline_pool is not None: + state.pipeline = self.pipeline_pool.row(state.slot) + def _grow_slots(self): """Double the slot table on the compute stream; live states move with it.""" old = self.slot_values @@ -442,17 +475,31 @@ def _grow_slots(self): # Work already queued still addresses the old storage. self._retired_slot_values.append(old) self.slot_values = new + if self.pipeline_pool is not None: + pool = self.pipeline_pool + grown = DevicePipelineState.zeros( + len(new), self.pipeline_width, pool.hidden.shape[2], pool.hidden.dtype, self.device + ) + for source, target in zip(pool.tensors(), grown.tensors()): + target[: len(old)].copy_(source) + self._retired_slot_values.extend(pool.tensors()) + self.pipeline_pool = grown self.free_slots = list(reversed(range(len(old), len(new)))) + self.free_slots for state in self.states.values(): - state.token, state.position = new[state.slot, 0], new[state.slot, 1] + self.bind_slot(state) - def _execute_recurrent_step(self, hidden, positions, tables, depth, allocations, active=None): + def device_metadata(self, positions, tables, depth, allocations, active=None, writes=None): + """Describe rows whose positions stay on the device. + + Inactive rows attend no keys. Only ``writes`` rows store KV; the + default is the active rows. + """ if isinstance(depth, torch.Tensor): table = tables.gather(1, depth[:, None, None].expand(-1, 1, tables.shape[2])).squeeze(1) else: table = tables[:, depth].contiguous() blocks = table.gather(1, (positions // self.cache.block_size)[:, None]).squeeze(1) - metadata = self.metadata.prepare_device( + return self.metadata.prepare_device( allocations=allocations, position_ids=positions, write_blocks=blocks.long(), @@ -461,8 +508,17 @@ def _execute_recurrent_step(self, hidden, positions, tables, depth, allocations, context_lengths=( (positions + 1) if active is None else torch.where(active, positions + 1, 0) ).int(), - write_mask=active, + write_mask=active if writes is None else writes, ) + + def boundary_metadata(self, positions, tables, allocations): + """PRELUDE and CODA rows of a boundary-KV model, which keep their history at depth 0.""" + if not self.boundary_kv: + return None + return self.device_metadata(positions, tables, 0, allocations) + + def _execute_recurrent_step(self, hidden, positions, tables, depth, allocations, active=None): + metadata = self.device_metadata(positions, tables, depth, allocations, active) forward = ForwardBatch( Operation.RECURRENT, hidden=hidden, metadata=(metadata,), graphs=self.graphs ) @@ -635,7 +691,8 @@ def _draft(self, plan, tables, allocations, u=None): row_tokens = tokens[active, offset] positions = starts[active] + offset row_tables = tables[active] - hidden = self.runtime.forward(ForwardBatch(Operation.PRELUDE, tokens=row_tokens)).hidden + boundary = self.boundary_metadata(positions, row_tables, allocations) + hidden = self._prelude(row_tokens, boundary) for depth in range(self.config.draft_loops): hidden = self._execute_recurrent_step( hidden, positions, row_tables, depth, allocations @@ -644,9 +701,13 @@ def _draft(self, plan, tables, allocations, u=None): if drafting_count: rows = getattr(plan, f"draft_rows{offset}", None) drafting = hidden if rows is None else hidden[rows] + if self.boundary_kv and rows is not None: + boundary = self.boundary_metadata( + positions[rows], row_tables[rows], allocations + ) # CUDA graphs may return a static buffer that the next _coda replay # overwrites: everything below is derived from `logits` immediately. - logits = self._coda(drafting) + logits = self._coda(drafting, boundary) proposed = logits.argmax(-1) random_rows = getattr(plan, f"r_rows{offset}", None) if random_rows is not None: @@ -676,7 +737,7 @@ def _verify(self, plan, starts, hidden_blocks, tables, allocations): hidden = torch.cat(hidden_blocks)[plan.hidden_rows] for depth in range(self.config.draft_loops, self.config.target_loops): hidden = self._execute_recurrent_step(hidden, positions, row_tables, depth, allocations) - logits = self._coda(hidden) + logits = self._coda(hidden, self.boundary_metadata(positions, row_tables, allocations)) target = logits.argmax(-1) random = plan.random if random is None: @@ -829,88 +890,116 @@ def submit(self, batch): ) def _pipeline_step(self, spans, states, tables, allocations, buffer): - widths = [ - min(span.token_count, len(state.pipeline.tokens)) for span, state in zip(spans, states) + pool, width = self.pipeline_pool, self.pipeline_width + n = len(spans) + # Budget reductions discard speculative descendants, never the root. + widths = np.array([min(span.token_count, width) for span in spans], dtype=np.int64) + # Row r of the step is slot column flat[r] % width of request requests[r], + # request-major and oldest first, as the per-request rows were before. + requests = np.repeat(np.arange(n), widths) + columns = np.arange(len(requests)) - np.repeat(np.cumsum(widths) - widths, widths) + limits = [ + len(span.request.prompt_token_ids) + span.request.sampling_params.max_tokens - 1 + for span in spans ] - active = torch.cat([state.pipeline.active[:w] for state, w in zip(states, widths)]) - depths = torch.cat([state.pipeline.depths[:w] for state, w in zip(states, widths)]) - positions = torch.cat( - [ - state.position + torch.arange(w, device=self.device) - for state, w in zip(states, widths) - ] - ) + host = torch.from_numpy( + np.concatenate( + ( + [state.slot for state in states], + widths, + limits, + requests, + requests * width + columns, + ) + ) + ).pin_memory() + values = host.to(self.device, non_blocking=True) + buffer.keepalive.extend((host, values)) + slots, w, limit, row_requests, flat = values.split((n, n, n, len(requests), len(requests))) + tokens, depths, retained, candidates, active = (t[slots] for t in pool.tensors()) + token, position = self.slot_values[slots].unbind(1) + ids = torch.arange(width, device=self.device) + row_active = active.view(-1)[flat] + row_depths = depths.view(-1)[flat] # Padding never reads beyond the reserved context or writes live KV. - positions = torch.where(active, positions, 0) - tokens = torch.cat([state.pipeline.tokens[:w] for state, w in zip(states, widths)]) - retained = torch.cat([state.pipeline.hidden[:w] for state, w in zip(states, widths)]) - prelude = self.runtime.forward(ForwardBatch(Operation.PRELUDE, tokens=tokens)).hidden - hidden = torch.where((depths == 0)[:, None], prelude, retained) - row_tables = torch.cat([tables[i : i + 1].expand(w, -1, -1) for i, w in enumerate(widths)]) + positions = torch.where(row_active, (position[:, None] + ids).view(-1)[flat], 0) + row_tables = tables[row_requests] + fresh = row_depths == 0 + prelude_kv = coda_kv = None + if self.boundary_kv: + # Every row runs both boundary stages to keep one call shape. A + # retained row already holds the input-stage KV of its token. + prelude_kv = self.device_metadata( + positions, row_tables, 0, allocations, row_active, row_active & fresh + ) + # Only rows that reach the draft or target depth read a distribution. + # The output-stage KV of other rows must stay the KV of their draft pass. + # Fresh rows reach the draft depth, so CODA writes the PRELUDE rows and + # the rows that reach the target depth. Both stages reuse one depth-0 + # view because eager rounds are bound by kernel launches. + last = row_depths == self.config.target_loops - self.config.draft_loops + coda_kv = replace(prelude_kv, write_mask=prelude_kv.write_mask | (row_active & last)) + prelude = self._prelude(tokens.view(-1)[flat], prelude_kv) + hidden = torch.where(fresh[:, None], prelude, retained.view(n * width, -1)[flat]) for offset in range(self.config.draft_loops): - depth = torch.where(active, depths + offset, 0) + depth = torch.where(row_active, row_depths + offset, 0) hidden = self._execute_recurrent_step( - hidden, positions, row_tables, depth, allocations, active + hidden, positions, row_tables, depth, allocations, row_active ) - choices = self._coda(hidden).argmax(-1) - start = 0 - for i, (span, state, w) in enumerate(zip(spans, states, widths)): - p = state.pipeline - ids = torch.arange(len(p.tokens), device=self.device) - # Budget reductions discard speculative descendants, never the root. - valid = p.active & (ids < w) - new_depths = p.depths + valid.long() * self.config.draft_loops - predictions = torch.zeros_like(p.tokens) - predictions[:w].copy_(choices[start : start + w]) - next_hidden = p.hidden.clone() - next_hidden[:w].copy_(hidden[start : start + w]) - drafted = valid & (new_depths == self.config.draft_loops) - candidates = torch.where(drafted, predictions, p.candidates) - emitted = valid[0] & (new_depths[0] == self.config.target_loops) - accepted = emitted & (candidates[0] == predictions[0]) - reject = emitted & ~accepted - state.token.copy_(torch.where(emitted, predictions[0], state.token)) - state.position.add_(emitted.long()) - # Shift only on target completion. All decisions remain stream-ordered tensors. - source = (ids + emitted.long()).clamp_max(len(ids) - 1) - remaining = valid.long().sum() - emitted.long() - surviving = (ids < remaining) & ~reject - p.tokens.copy_(p.tokens.gather(0, source)) - p.depths.copy_(new_depths.gather(0, source)) - p.hidden.copy_(next_hidden[source]) - p.candidates.copy_(candidates.gather(0, source)) - tail = (valid.long().sum() - 1).clamp_min(0) - append = drafted.gather(0, tail.reshape(1)).squeeze(0) & ~reject - newborn = (ids == remaining) & append - # A rejection (or a one-row budget) restarts from the corrected root. - restart = reject | (emitted & (remaining == 0)) - newborn = torch.where(restart, ids == 0, newborn) - limit = len(span.request.prompt_token_ids) + span.request.sampling_params.max_tokens - 1 - newborn &= (ids < w) & (state.position + ids < limit) - candidate = candidates.gather(0, tail.reshape(1)).squeeze(0) - token = torch.where(restart, state.token, candidate) - p.tokens.copy_(torch.where(newborn, token, p.tokens)) - p.depths.masked_fill_(newborn, 0) - p.active.copy_((surviving | newborn) & (state.position + ids < limit)) - # Per-request wire order matches SpeculativeRoundTicket.collect(): - # prediction, accepted, emitted, next_position, drafted, verified. - # A pipeline step verifies at most one token per request, so the - # verified count equals emitted (0 or 1). - buffer.result[6 * i : 6 * i + 6].copy_( - torch.stack( - ( - predictions[0], - accepted.long(), - emitted.long(), - state.position, - drafted.long().sum(), - emitted.long(), - ) - ) + choices = self._coda(hidden, coda_kv).argmax(-1) + # One update for all requests; every decision stays a stream-ordered tensor. + valid = active & (ids < w[:, None]) + new_depths = depths + valid.long() * self.config.draft_loops + predictions = torch.zeros_like(tokens) + predictions.view(-1)[flat] = choices + retained.view(n * width, -1)[flat] = hidden + drafted = valid & (new_depths == self.config.draft_loops) + candidates = torch.where(drafted, predictions, candidates) + emitted = valid[:, 0] & (new_depths[:, 0] == self.config.target_loops) + accepted = emitted & (candidates[:, 0] == predictions[:, 0]) + reject = emitted & ~accepted + token = torch.where(emitted, predictions[:, 0], token) + position = position + emitted.long() + # Shift only on target completion. + source = (ids + emitted.long()[:, None]).clamp_max(width - 1) + count = valid.long().sum(1) + remaining = count - emitted.long() + surviving = (ids < remaining[:, None]) & ~reject[:, None] + tail = (count - 1).clamp_min(0)[:, None] + append = drafted.gather(1, tail).squeeze(1) & ~reject + newborn = (ids == remaining[:, None]) & append[:, None] + # A rejection (or a one-row budget) restarts from the corrected root. + restart = reject | (emitted & (remaining == 0)) + newborn = torch.where(restart[:, None], ids == 0, newborn) + inside = position[:, None] + ids < limit[:, None] + newborn &= (ids < w[:, None]) & inside + start = torch.where(restart, token, candidates.gather(1, tail).squeeze(1)) + pool.tokens[slots] = torch.where(newborn, start[:, None], tokens.gather(1, source)) + pool.depths[slots] = new_depths.gather(1, source).masked_fill(newborn, 0) + pool.hidden[slots] = retained.gather(1, source[:, :, None].expand_as(retained)) + pool.candidates[slots] = candidates.gather(1, source) + pool.active[slots] = (surviving | newborn) & inside + self.slot_values[slots] = torch.stack((token, position), 1) + # Per-request wire order matches SpeculativeRoundTicket.collect(): + # prediction, accepted, emitted, next_position, drafted, verified. + # A pipeline step verifies at most one token per request, so the + # verified count equals emitted (0 or 1). + buffer.result[: 6 * n].view(n, 6).copy_( + torch.stack( + ( + predictions[:, 0], + accepted.long(), + emitted.long(), + position, + drafted.long().sum(1), + emitted.long(), + ), + 1, ) + ) + for state in states: state.submitted += 1 - start += w - return start + return len(requests) def snapshot_pipeline(self, request, device): state = self.states.get(request.request_id) @@ -950,44 +1039,40 @@ def restore_pipeline(self, request, snapshot): raise RuntimeError("invalid pipeline restoration generation") if snapshot.pending or snapshot.position != request.position: raise RuntimeError("pipeline snapshot disagrees with committed request position") - width = min( - self.config.num_speculative_tokens + 1, - self.config.target_loops // self.config.draft_loops, - ) + width = self.pipeline_width if len(snapshot.tokens) > width: raise RuntimeError("pipeline snapshot exceeds the configured window") with torch.cuda.stream(self.compute_stream): + if not self.free_slots: + self._grow_slots() + slot = self.free_slots.pop() + pool = self.pipeline_pool + padding = [0] * (width - len(snapshot.tokens)) values = torch.tensor( [ - [t.token for t in snapshot.tokens] + [0] * (width - len(snapshot.tokens)), - [t.depth for t in snapshot.tokens] + [0] * (width - len(snapshot.tokens)), - [t.candidate or 0 for t in snapshot.tokens] - + [0] * (width - len(snapshot.tokens)), + [t.token for t in snapshot.tokens] + padding, + [t.depth for t in snapshot.tokens] + padding, + [t.candidate or 0 for t in snapshot.tokens] + padding, ], device=self.device, ) - hidden = torch.zeros( - (width, self.model.config.hidden_size), - device=self.device, - dtype=next(self.model.parameters()).dtype, - ) + pool.tokens[slot], pool.depths[slot], pool.candidates[slot] = values + pool.hidden[slot].zero_() for i, token in enumerate(snapshot.tokens): if token.hidden is not None: - hidden[i].copy_(token.hidden) - pipeline = DevicePipelineState( - values[0], - values[1], - hidden, - values[2], - torch.arange(width, device=self.device) < len(snapshot.tokens), + pool.hidden[slot, i].copy_(token.hidden) + pool.active[slot] = torch.arange(width, device=self.device) < len(snapshot.tokens) + self.slot_values[slot] = torch.tensor( + [request.input_token_id, snapshot.position], device=self.device ) - self.states[request.request_id] = SpeculativeRequestState( + state = SpeculativeRequestState( request, self.cache._get_allocation(request.request_id), - torch.tensor(request.input_token_id, device=self.device), - torch.tensor(snapshot.position, device=self.device), - pipeline, + *self.slot_values[slot].unbind(), + pool.row(slot), + slot=slot, ) + self.states[request.request_id] = state def request_ready(self, request): state = self.states.get(request.request_id) diff --git a/vllm_rlt/worker/cuda_graph.py b/vllm_rlt/worker/cuda_graph.py index c0f0ceac..053ca9d7 100644 --- a/vllm_rlt/worker/cuda_graph.py +++ b/vllm_rlt/worker/cuda_graph.py @@ -9,6 +9,7 @@ import torch from vllm_rlt.config import cuda_graph_rows +from vllm_rlt.request import Operation from vllm_rlt.worker.metadata import DeviceKVMetadata logger = logging.getLogger(__name__) @@ -534,3 +535,76 @@ def _run(self, entry): def _eager(self, tokens, batch): return self.model.prelude_prepared(tokens, batch, self.cache) + + +class DeviceBoundaryGraphs: + """Bucketed input- or output-stage graphs of a boundary-KV model. + + Asynchronous speculation keeps row positions on the device, so positions, + write addresses, write masks and context lengths stay graph inputs. Entries + hold no request allocation and never touch CPU written-position state. + """ + + def __init__(self, model, cache, operation, capture_sizes): + self.cache = cache + self.capture_sizes = capture_sizes + self.max_positions = model.config.max_position_embeddings + if operation == Operation.PRELUDE: + self.stage = model.prelude_prepared + self.example = torch.empty(0, dtype=torch.long, device=cache.device) + else: + self.stage = model.coda_prepared + self.example = next(model.parameters()).new_empty((0, model.config.hidden_size)) + self.entries = {} + self.pool = torch.cuda.graph_pool_handle() + self.stream = torch.cuda.Stream(device=cache.device) + self.captures = self.replays = self.fallbacks = 0 + + @torch.inference_mode() + def prepare(self): + """Capture every bucket before any serving request exists.""" + device = self.cache.device + batch = SimpleNamespace( + position_ids=torch.empty(0, device=device, dtype=torch.long), + write_blocks=torch.empty(0, device=device, dtype=torch.long), + write_offsets=torch.empty(0, device=device, dtype=torch.long), + context_lengths=torch.empty(0, device=device, dtype=torch.int32), + block_tables=torch.empty((0, 1), device=device, dtype=torch.int32), + cu_seqlens_q=None, + ) + proxy = _DeviceCache(self.cache) + for size in self.capture_sizes: + entry = _graph_entry( + self.example, batch, self.cache, self.max_positions, count=size, bucketed=True + ) + _copy_graph_inputs(entry, self.example, batch, count=0) + entry.graph, entry.output = _capture( + entry.hidden, + self.stream, + self.pool, + lambda: self.stage(entry.hidden, entry.metadata, proxy), + startup=True, + ) + self.entries[size] = entry + self.captures += 1 + + def select(self, inputs, batch): + count = _query_count(batch) + if count > self.capture_sizes[-1]: + self.fallbacks += 1 + return PreparedGraph(batch) + return PreparedGraph(batch, self.entries[_bucket_size(self.capture_sizes, count)]) + + @torch.inference_mode() + def prepare_inputs(self, call, inputs): + if call.entry is not None: + _copy_graph_inputs(call.entry, inputs, call.metadata) + + @torch.inference_mode() + def execute(self, call, inputs): + if call.entry is None: + return self.stage(inputs, call.metadata, self.cache) + call.entry.graph.replay() + self.replays += 1 + # The next replay of this bucket overwrites the static output. + return call.entry.output[: _query_count(call.metadata)].clone() diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index d292e422..9a1ee2d1 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -20,7 +20,7 @@ from vllm_rlt.core.scheduler import SchedulerOutput from vllm_rlt.request import Operation, Request from vllm_rlt.worker.buffers import stage_metadata -from vllm_rlt.worker.cuda_graph import CodaGraphs, RecurrentGraphs +from vllm_rlt.worker.cuda_graph import CodaGraphs, PreludeGraphs, RecurrentGraphs from vllm_rlt.worker.executor import ForwardBatch, ModelRuntime from vllm_rlt.worker.metadata import KVMetadata from vllm_rlt.worker.output import ModelRunnerOutput, SpeculativeResult @@ -216,7 +216,13 @@ def __init__( self.graphs = ( RecurrentGraphs(model, cache, execution, False) if execution.cuda_graphs else None ) - self.coda_graphs = CodaGraphs(model, execution) if execution.cuda_graphs else None + self.coda_graphs = self.prelude_graphs = None + if execution.cuda_graphs and self.boundary_kv: + # Boundary stages attend KV, so their graphs take KV metadata as input. + self.coda_graphs = CodaGraphs(model, execution, cache) + self.prelude_graphs = PreludeGraphs(model, execution, cache) + elif execution.cuda_graphs: + self.coda_graphs = CodaGraphs(model, execution) def _packed(self, packed): capabilities = self.cache.attention_capabilities @@ -315,7 +321,14 @@ def _prepare_inputs(self, batch): def _prelude(self, tokens, boundary=None): return self.runtime.forward( - ForwardBatch(Operation.PRELUDE, tokens=tokens, prelude_coda_kv_metadata=boundary) + self.runtime.prepare( + ForwardBatch( + Operation.PRELUDE, + tokens=tokens, + prelude_coda_kv_metadata=boundary, + graphs=self.prelude_graphs, + ) + ) ).hidden def _recurrent(self, hidden, metadata):