From b3d96ba931294bdcbe9b0d8e39d9084e00a10675 Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Tue, 17 Nov 2026 23:54:09 -0500 Subject: [PATCH 1/7] feat(dory): run asynchronous self-speculation with boundary KV Asynchronous block and pipeline rounds now give PRELUDE and CODA their depth-0 KV. The positions of these rows exist only on the device. Thus the runner builds device metadata for the boundary stages, as it does for the recurrent loops. A pipeline step runs both boundary stages on all rows. Write masks keep the input-stage KV of retained rows and the output-stage KV of rows between the draft and target depths. Each step stores the same KV as a synchronous step. The engine accepts asynchronous eager rounds for boundary-KV models. CUDA Graphs, preemption and sampled requests stay rejected. --- tests/test_async_speculative.py | 4 +- tests/test_async_speculative_graphs.py | 6 +- tests/test_async_speculative_lifecycle.py | 4 +- tests/test_async_speculative_sampling_plan.py | 3 +- tests/test_dory_speculative.py | 2 +- tests/test_dory_speculative_gpu.py | 410 ++++++++++++++++++ tests/test_speculative.py | 2 +- vllm_rlt/engine/stage_engine.py | 25 +- vllm_rlt/worker/async_speculative.py | 57 ++- 9 files changed, 484 insertions(+), 29 deletions(-) create mode 100644 tests/test_dory_speculative_gpu.py diff --git a/tests/test_async_speculative.py b/tests/test_async_speculative.py index e777e967..2b630a54 100644 --- a/tests/test_async_speculative.py +++ b/tests/test_async_speculative.py @@ -132,7 +132,7 @@ def test_second_round_executes_before_cpu_commit_at_every_rejection(accepted, mo start = request.position calls = 0 - def controlled(hidden): + def controlled(hidden, boundary=None): nonlocal calls phase = calls % 4 calls += 1 @@ -182,7 +182,7 @@ def test_eos_discards_queued_round_and_request_id_can_be_reused(monkeypatch, gra monkeypatch.setattr( e.speculative_runner, "_coda", - lambda h: torch.zeros( + lambda h, boundary=None: torch.zeros( len(h), model.config.vocab_size, device="cuda", diff --git a/tests/test_async_speculative_graphs.py b/tests/test_async_speculative_graphs.py index 04ef90e5..67adb2f0 100644 --- a/tests/test_async_speculative_graphs.py +++ b/tests/test_async_speculative_graphs.py @@ -99,7 +99,7 @@ def test_two_graph_rounds_keep_device_frontier_and_rejection_mask(backend, accep written_before = [[layer.prefix for layer in depth] for depth in allocation.written] calls = 0 - def controlled(hidden): + def controlled(hidden, boundary=None): nonlocal calls phase = calls % 4 calls += 1 @@ -165,7 +165,9 @@ def test_graph_eos_retires_later_round_before_id_reuse(backend, monkeypatch): patch.setattr( runner, "_coda", - lambda h: torch.zeros(len(h), engine.model.config.vocab_size, device=h.device), + lambda h, boundary=None: torch.zeros( + len(h), engine.model.config.vocab_size, device=h.device + ), ) engine.step() engine.step() diff --git a/tests/test_async_speculative_lifecycle.py b/tests/test_async_speculative_lifecycle.py index bba1076b..9b1c24b9 100644 --- a/tests/test_async_speculative_lifecycle.py +++ b/tests/test_async_speculative_lifecycle.py @@ -164,7 +164,9 @@ def test_cuda_serving_two_completed_rounds_isolate_decode_failure( monkeypatch.setattr( engine.speculative_runner, "_coda", - lambda h: torch.zeros(len(h), engine.model.config.vocab_size, device=h.device), + lambda h, boundary=None: torch.zeros( + len(h), engine.model.config.vocab_size, device=h.device + ), ) engine.step() engine.step() diff --git a/tests/test_async_speculative_sampling_plan.py b/tests/test_async_speculative_sampling_plan.py index 002aa439..fd11991a 100644 --- a/tests/test_async_speculative_sampling_plan.py +++ b/tests/test_async_speculative_sampling_plan.py @@ -207,6 +207,7 @@ def __init__(self, monkeypatch, seed): num_speculative_tokens=self.K, draft_loops=1, target_loops=2 ) runner.device = torch.device("cpu") + runner.boundary_kv = False runner.rng = RngRegistry() runner.slot_values = torch.zeros((8, 2), dtype=torch.long) runner.runtime = SimpleNamespace( @@ -215,7 +216,7 @@ def __init__(self, monkeypatch, seed): runner._execute_recurrent_step = lambda hidden, positions, tables, depth, allocations: ( hidden if depth < 1 else hidden @ self.M ) - runner._coda = lambda hidden: hidden @ self.W + runner._coda = lambda hidden, boundary=None: hidden @ self.W self.runner = runner def run(self, specs): diff --git a/tests/test_dory_speculative.py b/tests/test_dory_speculative.py index e52b2832..be5e4877 100644 --- a/tests/test_dory_speculative.py +++ b/tests/test_dory_speculative.py @@ -426,7 +426,7 @@ def distribution(tokens): def test_unqualified_speculative_combinations_fail_at_construction(): model = dory(2) - with pytest.raises(ValueError, match="synchronous"): + with pytest.raises(ValueError, match="requires CUDA"): engine(model, 2, 1, execution_config=ExecutionConfig(async_scheduling=True)) with pytest.raises(ValueError, match="CUDA graphs"): engine(model, 2, 1, execution_config=ExecutionConfig(cuda_graphs=True)) diff --git a/tests/test_dory_speculative_gpu.py b/tests/test_dory_speculative_gpu.py new file mode 100644 index 00000000..007a3ea6 --- /dev/null +++ b/tests/test_dory_speculative_gpu.py @@ -0,0 +1,410 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Asynchronous Dory self-speculation on CUDA: block and token-pipeline rounds. + +Every case needs an explicit ``CUDA_VISIBLE_DEVICES`` and ``--run-gpu``. Each +mode must serve the greedy tokens of the dense oracle. Forced drafts check that +the committed KV of every plane is the serial target history while the next +round is already submitted. +""" + +import pytest +import torch + +from tests.reference.dory import dense_dory_reference +from tests.test_dory import make_cache, tiny_dory_config +from vllm_rlt import ( + CacheConfig, + ExecutionConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.models import DoryForCausalLM + +pytestmark = pytest.mark.gpu + +DEPTH_PAIRS = [(1, 2), (1, 4), (2, 4)] +# Prompts shorter than and longer than the tiny window of four. +PROMPTS = {"a": [4, 7, 3], "b": [9], "c": [5, 6, 8, 2, 11]} +LIMITS = {"a": 12, "b": 1, "c": 7} + + +def reference_pair(loops, **overrides): + """A CPU oracle model and a CUDA model with the same FP32 weights.""" + torch.manual_seed(42) + reference = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops, **overrides)).eval() + model = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops, **overrides)) + model.load_state_dict(reference.state_dict()) + return reference, model.to("cuda").eval() + + +def engine(model, k, draft, mode="block", *, asynchronous=True): + return LLMEngine( + model, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=1024, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=4), + execution_config=ExecutionConfig(async_scheduling=asynchronous), + speculative_config=SpeculativeConfig( + k, draft_loops=draft, target_loops=model.config.total_ut_steps, mode=mode + ), + ) + + +def params(model, max_tokens, **kwargs): + loops = model.config.total_ut_steps + kwargs.setdefault("ignore_eos", True) + return SamplingParams(max_tokens=max_tokens, min_loops=loops, max_loops=loops, **kwargs) + + +def drain(e): + final = {} + for _ in range(4000): + if not e.has_unfinished_requests(): + return final + for output in e.step(): + if output.finished: + final[output.request_id] = output + pytest.fail("speculative scheduler did not drain") + + +def greedy_reference(model, prompt, count): + tokens = list(prompt) + for _ in range(count): + logits = dense_dory_reference(model, torch.tensor(tokens))["logits"] + tokens.append(int(logits[-1].argmax())) + return tokens + + +def assert_released(e): + runner = e.speculative_runner + assert not runner.states + assert all(not buffer.leased for buffer in runner.buffers) + assert e.cache_manager.num_used_blocks == 0 + + +def serial_cache(reference, tokens): + """KV of the reference schedule: one token per step at the full depth.""" + cache = make_cache(reference) + tokens = torch.tensor(tokens) + assert cache.allocate("request", len(tokens)) + for position in range(len(tokens)): + boundary = cache._prepare_batch(["request"], [0], [position]) + hidden = reference.prelude_prepared(tokens[position : position + 1], boundary, cache) + for depth in range(reference.config.total_ut_steps): + core = cache._prepare_batch(["request"], [depth], [position]) + hidden, _ = reference.recurrent_prepared(hidden, core, cache) + reference.coda_prepared(hidden, boundary, cache) + return cache + + +def device_kv(cache, layer, request_id, depth, length): + """Read KV that device-owned rounds wrote; the host written frontier lags it.""" + table = torch.tensor(cache.get_block_table(request_id, depth), dtype=torch.long) + positions = torch.arange(length) + blocks = table[positions // cache.block_size].to(cache.device) + offsets = (positions % cache.block_size).to(cache.device) + return cache.key_cache[blocks, layer, offsets], cache.value_cache[blocks, layer, offsets] + + +def assert_committed_kv_is_the_target_history(e, reference, request_id, tokens, frontier): + """Every plane below ``frontier`` holds what serial execution stores.""" + e.speculative_runner.synchronize() + expected = serial_cache(reference, tokens[:frontier]) + boundary = (*reference.prelude_kv_layers, *reference.coda_kv_layers) + for layer in range(reference.config.num_kv_layers): + depths = (0,) if layer in boundary else range(reference.config.total_ut_steps) + for depth in depths: + actual = device_kv(e.cache_manager, layer, request_id, depth, frontier) + wanted = expected.read(layer, "request", depth, frontier) + for a, b in zip(actual, wanted): + torch.testing.assert_close(a.cpu(), b, atol=2e-5, rtol=2e-4) + + +def force_drafts(monkeypatch, e, sequence, wrong): + """Make every draft row propose the reference token, or miss where ``wrong`` says. + + ``wrong(position, index)`` receives the input position of the draft row and, + in block mode, its draft offset in the round. Target rows keep the model logits. + The position read is a host synchronization that only this test adds. + """ + runner = e.speculative_runner + coda, draft = runner._coda, runner._draft + vocab = e.model.config.vocab_size + state = {"offset": None} + + def drafting(*args): + state["offset"] = 0 + try: + return draft(*args) + finally: + state["offset"] = None + + def controlled(hidden, boundary=None): + logits = coda(hidden, boundary) + positions = boundary.position_ids.tolist() + if runner.config.mode == "block": + if state["offset"] is None: + return logits + rows = [(row, state["offset"]) for row in range(len(positions))] + state["offset"] += 1 + else: + pipeline = runner.states["r"].pipeline + width = len(positions) + # The depths still describe the rows before this step. + depths = pipeline.depths[:width].tolist() + active = pipeline.active[:width].tolist() + rows = [(row, None) for row in range(width) if active[row] and depths[row] == 0] + for row, index in rows: + position = positions[row] + if position + 1 >= len(sequence): + continue + token = sequence[position + 1] + if wrong(position, index): + token = (token + 1) % vocab + logits[row] = -100.0 + logits[row, token] = 100.0 + return logits + + monkeypatch.setattr(runner, "_draft", drafting) + monkeypatch.setattr(runner, "_coda", controlled) + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("k", [1, 3]) +def test_async_greedy_matches_the_dense_reference(mode, draft, target, k): + reference, model = reference_pair(target) + e = engine(model, k, draft, mode) + for rid, prompt in PROMPTS.items(): + e.add_request(rid, prompt, params(model, LIMITS[rid])) + final = drain(e) + for rid, prompt in PROMPTS.items(): + expected = greedy_reference(reference, prompt, LIMITS[rid])[len(prompt) :] + assert final[rid].token_ids == expected + assert final[rid].exit_depths == [target] * LIMITS[rid] + assert e.speculative_runner.submitted_before_collect > 0 + assert_released(e) + + +def bootstrap(e, model, prompt, count): + e.add_request("r", prompt, params(model, count)) + request = e.scheduler.requests["r"] + for _ in range(30): + if e.step(): + return request + pytest.fail("bootstrap did not produce a token") + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_submission_reads_no_device_value(mode, monkeypatch): + _, model = reference_pair(4) + e = engine(model, 3, 1, mode) + request = bootstrap(e, model, [4, 7, 3], 16) + + def forbidden(*args, **kwargs): + raise AssertionError("host read in cross-round GPU submission") + + with monkeypatch.context() as patch: + patch.setattr(torch.Tensor, "item", forbidden) + patch.setattr(torch.Tensor, "tolist", forbidden) + e.step() + e.step() + assert len(e._pending_speculative) == 2 + assert len(request.generated_token_ids) == 1 + e.abort_request("r") + drain(e) + assert_released(e) + + +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("rejected", [0, 1, 2, 3]) +def test_block_commits_target_history_through_each_rejection(draft, target, rejected, monkeypatch): + """rejected=3 accepts all three candidates and commits the bonus token.""" + reference, model = reference_pair(target) + prompt, count = [4, 7, 3], 13 + sequence = greedy_reference(reference, prompt, count) + e = engine(model, 3, draft) + force_drafts(monkeypatch, e, sequence, lambda position, index: index == rejected) + e.add_request("r", prompt, params(model, count)) + request = e.scheduler.requests["r"] + overlapped = False + while e.has_unfinished_requests(): + outputs = e.step() + overlapped |= len(e._pending_speculative) == 2 + generated = list(request.generated_token_ids) + assert generated == sequence[len(prompt) : len(prompt) + len(generated)] + if outputs and outputs[0].finished: + break + if generated and e.cache_manager.has_allocation("r"): + frontier = len(prompt) + len(generated) - 1 + assert_committed_kv_is_the_target_history(e, reference, "r", sequence, frontier) + drain(e) + assert overlapped + stats = e.speculative_runner.stats + if rejected: + assert stats.accepted_tokens > 0 + else: + assert stats.accepted_tokens == 0 + assert request.generated_token_ids == sequence[len(prompt) :] + assert_released(e) + + +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +@pytest.mark.parametrize("pattern", ["all", "none", "every_third"]) +def test_pipeline_commits_target_history_under_forced_acceptance( + draft, target, pattern, monkeypatch +): + reference, model = reference_pair(target) + prompt, count = [4, 7, 3], 12 + sequence = greedy_reference(reference, prompt, count) + wrong = { + "all": lambda position, index: False, + "none": lambda position, index: True, + "every_third": lambda position, index: position % 3 == 0, + }[pattern] + e = engine(model, 3, draft, "pipeline") + force_drafts(monkeypatch, e, sequence, wrong) + e.add_request("r", prompt, params(model, count)) + request = e.scheduler.requests["r"] + while e.has_unfinished_requests(): + outputs = e.step() + generated = list(request.generated_token_ids) + assert generated == sequence[len(prompt) : len(prompt) + len(generated)] + if outputs and outputs[0].finished: + break + if generated and e.cache_manager.has_allocation("r"): + frontier = len(prompt) + len(generated) - 1 + assert_committed_kv_is_the_target_history(e, reference, "r", sequence, frontier) + drain(e) + stats = e.speculative_runner.stats + verified = count - 1 + assert stats.committed_tokens == verified + if pattern == "all": + assert stats.accepted_tokens == verified + elif pattern == "none": + assert stats.accepted_tokens == 0 + else: + assert 0 < stats.accepted_tokens < verified + assert_released(e) + + +def stage_kv(e, layers, position): + """KV of one position in every layer of one boundary stage.""" + return torch.stack( + [ + torch.stack(device_kv(e.cache_manager, layer, "r", 0, position + 1))[:, position] + for layer in layers + ] + ) + + +@pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) +def test_pipeline_keeps_the_boundary_kv_of_rows_in_flight(draft, target, monkeypatch): + """A pipeline step stores boundary KV only for the rows that use that stage. + + A retained row keeps the input-stage KV of its first pass. A row between + the draft and target depths keeps the output-stage KV of its draft pass, + which the draft rows behind it attend. + """ + _, model = reference_pair(target, ngpt_alpha_init=1.0) + e = engine(model, 3, draft, "pipeline") + runner = e.speculative_runner + coda = runner._coda + steps = [] + + def recording(hidden, boundary=None): + # Inside the step, the device depths still describe the rows before it. + pipeline = runner.states["r"].pipeline + positions = boundary.position_ids.tolist() + depths = pipeline.depths[: len(positions)].tolist() + active = pipeline.active[: len(positions)].tolist() + steps.append([(p, d) for p, d, a in zip(positions, depths, active) if a]) + return coda(hidden, boundary) + + monkeypatch.setattr(runner, "_coda", recording) + e.add_request("r", [4, 7, 3], params(model, 24)) + first, drafted = {}, {} + retained = between = 0 + while e.has_unfinished_requests(): + submitted = len(steps) + e.step() + if len(steps) == submitted or not e.cache_manager.has_allocation("r"): + continue + runner.synchronize() + for position, depth in steps[-1]: + prelude = stage_kv(e, model.prelude_kv_layers, position) + output = stage_kv(e, model.coda_kv_layers, position) + if depth == 0: + first[position], drafted[position] = prelude, output + continue + assert torch.equal(prelude, first[position]) + retained += 1 + if depth + draft < target: + assert torch.equal(output, drafted[position]) + between += 1 + assert retained + assert bool(between) == (target - draft > draft) + assert_released(e) + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_cancellation_with_rounds_in_flight_allows_id_reuse(mode): + reference, model = reference_pair(4) + e = engine(model, 3, 2, mode) + e.add_request("a", [4, 7, 3], params(model, 14)) + e.add_request("b", [9, 5], params(model, 8)) + final = {} + for _ in range(30): + final.update({output.request_id: output for output in e.step() if output.finished}) + if len(e._pending_speculative) == 2: + break + assert len(e._pending_speculative) == 2 and "a" not in final + assert e.abort_request("a").finish_reason == "abort" + e.add_request("a", [6, 2], params(model, 5)) + final.update(drain(e)) + assert final["a"].token_ids == greedy_reference(reference, [6, 2], 5)[2:] + assert final["b"].token_ids == greedy_reference(reference, [9, 5], 8)[2:] + assert_released(e) + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_failed_round_releases_requests_and_allows_id_reuse(mode, monkeypatch): + reference, model = reference_pair(2) + e = engine(model, 3, 1, mode) + bootstrap(e, model, [4, 7, 3], 8) + runner = e.speculative_runner + name = "_pipeline_step" if mode == "pipeline" else "_verify" + original = getattr(runner, name) + + def fail(*args): + original(*args) + raise RuntimeError("injected failure after submission") + + monkeypatch.setattr(runner, name, fail) + with pytest.raises(RuntimeError, match="injected failure"): + drain(e) + assert not e._pending_speculative and not e._retiring_speculative + assert not e.scheduler.requests + assert_released(e) + monkeypatch.setattr(runner, name, original) + e.add_request("r", [5], params(model, 3)) + assert drain(e)["r"].token_ids == greedy_reference(reference, [5], 3)[1:] + assert_released(e) + + +def test_unqualified_async_combinations_are_rejected(): + _, model = reference_pair(2) + e = engine(model, 2, 1) + with pytest.raises(ValueError, match="greedy sampling"): + e.add_request("r", [4], params(model, 2, temperature=0.7)) + with pytest.raises(ValueError, match="preemption"): + LLMEngine( + model, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=4, enable_preemption=True), + execution_config=ExecutionConfig(async_scheduling=True), + speculative_config=SpeculativeConfig(2, draft_loops=1, target_loops=2), + ) diff --git a/tests/test_speculative.py b/tests/test_speculative.py index 7efb6992..a18cb79b 100644 --- a/tests/test_speculative.py +++ b/tests/test_speculative.py @@ -1207,7 +1207,7 @@ def test_async_pipeline_uses_gpu_decision_before_collect(reject, monkeypatch): runner = e.speculative_runner calls = 0 - def controlled(hidden): + def controlled(hidden, boundary=None): nonlocal calls calls += 1 logits = torch.full( diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index ce700908..5473a564 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -99,16 +99,12 @@ def __init__( if speculative_config is not None: if not getattr(model, "supports_self_speculation", False): raise ValueError("boundary-KV speculative decoding is not yet supported") - # Qualified: synchronous eager rounds. Async rounds, Graph replay and + # Qualified: synchronous and asynchronous eager rounds. Graph replay and # resumable preemption of draft/target boundary state are separate work. - if ( - self.execution_config.async_scheduling - or self.execution_config.cuda_graphs - or scheduler_config.enable_preemption - ): + if self.execution_config.cuda_graphs or scheduler_config.enable_preemption: raise ValueError( - "boundary-KV speculative decoding requires synchronous eager " - "execution without CUDA graphs or preemption" + "boundary-KV speculative decoding requires eager execution " + "without CUDA graphs or preemption" ) if not getattr(model, "supports_cuda_graphs", True) and self.execution_config.cuda_graphs: raise ValueError("this model does not support CUDA graphs yet") @@ -362,10 +358,15 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i if not isinstance(request_id, str) or not request_id: raise ValueError("request_id must be a nonempty string") params = sampling_params or SamplingParams() - if ( - self.speculative_config and self.speculative_config.mode == "pipeline" - ) and params.temperature != 0: - raise ValueError("pipeline speculative decoding currently requires greedy sampling") + if params.temperature != 0: + if self.pipeline_speculative: + raise ValueError("pipeline speculative decoding currently requires greedy sampling") + # RFC #176 qualifies asynchronous boundary-KV rounds for greedy requests only. + if self.async_speculative and getattr(self.model, "requires_boundary_kv", False): + raise ValueError( + "asynchronous boundary-KV speculative decoding currently requires " + "greedy sampling" + ) if request_id in self._retiring_speculative: self._retire_speculative(wait=True, request_id=request_id) config = self.model.config diff --git a/vllm_rlt/worker/async_speculative.py b/vllm_rlt/worker/async_speculative.py index 8b18dfa2..decff876 100644 --- a/vllm_rlt/worker/async_speculative.py +++ b/vllm_rlt/worker/async_speculative.py @@ -5,6 +5,11 @@ verify with speculative rejection sampling against ``p`` (see ``speculative_sampling``) and return their committed sequence (accepted drafts, then correction or bonus) in the result rows. A batch without random requests takes the greedy path with no extra device work. + +Boundary-KV models (attention in PRELUDE and CODA) keep the history of both stages at +depth 0. The contract is the same as in ``speculative``: all KV below the start of the +next round is the serial target history. The boundary stages use device metadata, as +the recurrent loops do, because the actual positions exist only on the device. """ import math @@ -446,13 +451,18 @@ def _grow_slots(self): for state in self.states.values(): state.token, state.position = new[state.slot, 0], new[state.slot, 1] - def _execute_recurrent_step(self, hidden, positions, tables, depth, allocations, active=None): + def device_metadata(self, positions, tables, depth, allocations, active=None, writes=None): + """Describe rows whose positions stay on the device. + + Inactive rows attend no keys. Only ``writes`` rows store KV; the + default is the active rows. + """ if isinstance(depth, torch.Tensor): table = tables.gather(1, depth[:, None, None].expand(-1, 1, tables.shape[2])).squeeze(1) else: table = tables[:, depth].contiguous() blocks = table.gather(1, (positions // self.cache.block_size)[:, None]).squeeze(1) - metadata = self.metadata.prepare_device( + return self.metadata.prepare_device( allocations=allocations, position_ids=positions, write_blocks=blocks.long(), @@ -461,8 +471,17 @@ def _execute_recurrent_step(self, hidden, positions, tables, depth, allocations, context_lengths=( (positions + 1) if active is None else torch.where(active, positions + 1, 0) ).int(), - write_mask=active, + write_mask=active if writes is None else writes, ) + + def boundary_metadata(self, positions, tables, allocations): + """PRELUDE and CODA rows of a boundary-KV model, which keep their history at depth 0.""" + if not self.boundary_kv: + return None + return self.device_metadata(positions, tables, 0, allocations) + + def _execute_recurrent_step(self, hidden, positions, tables, depth, allocations, active=None): + metadata = self.device_metadata(positions, tables, depth, allocations, active) forward = ForwardBatch( Operation.RECURRENT, hidden=hidden, metadata=(metadata,), graphs=self.graphs ) @@ -635,7 +654,8 @@ def _draft(self, plan, tables, allocations, u=None): row_tokens = tokens[active, offset] positions = starts[active] + offset row_tables = tables[active] - hidden = self.runtime.forward(ForwardBatch(Operation.PRELUDE, tokens=row_tokens)).hidden + boundary = self.boundary_metadata(positions, row_tables, allocations) + hidden = self._prelude(row_tokens, boundary) for depth in range(self.config.draft_loops): hidden = self._execute_recurrent_step( hidden, positions, row_tables, depth, allocations @@ -644,9 +664,13 @@ def _draft(self, plan, tables, allocations, u=None): if drafting_count: rows = getattr(plan, f"draft_rows{offset}", None) drafting = hidden if rows is None else hidden[rows] + if self.boundary_kv and rows is not None: + boundary = self.boundary_metadata( + positions[rows], row_tables[rows], allocations + ) # CUDA graphs may return a static buffer that the next _coda replay # overwrites: everything below is derived from `logits` immediately. - logits = self._coda(drafting) + logits = self._coda(drafting, boundary) proposed = logits.argmax(-1) random_rows = getattr(plan, f"r_rows{offset}", None) if random_rows is not None: @@ -676,7 +700,7 @@ def _verify(self, plan, starts, hidden_blocks, tables, allocations): hidden = torch.cat(hidden_blocks)[plan.hidden_rows] for depth in range(self.config.draft_loops, self.config.target_loops): hidden = self._execute_recurrent_step(hidden, positions, row_tables, depth, allocations) - logits = self._coda(hidden) + logits = self._coda(hidden, self.boundary_metadata(positions, row_tables, allocations)) target = logits.argmax(-1) random = plan.random if random is None: @@ -844,15 +868,30 @@ def _pipeline_step(self, spans, states, tables, allocations, buffer): positions = torch.where(active, positions, 0) tokens = torch.cat([state.pipeline.tokens[:w] for state, w in zip(states, widths)]) retained = torch.cat([state.pipeline.hidden[:w] for state, w in zip(states, widths)]) - prelude = self.runtime.forward(ForwardBatch(Operation.PRELUDE, tokens=tokens)).hidden - hidden = torch.where((depths == 0)[:, None], prelude, retained) row_tables = torch.cat([tables[i : i + 1].expand(w, -1, -1) for i, w in enumerate(widths)]) + fresh = depths == 0 + prelude_kv = coda_kv = None + if self.boundary_kv: + # Every row runs both boundary stages to keep one call shape. A + # retained row already holds the input-stage KV of its token. + prelude_kv = self.device_metadata( + positions, row_tables, 0, allocations, active, active & fresh + ) + # Only rows that reach the draft or target depth read a distribution. + # The output-stage KV of other rows must stay the KV of their draft pass. + reached = depths + self.config.draft_loops + endpoints = (reached == self.config.draft_loops) | (reached == self.config.target_loops) + coda_kv = self.device_metadata( + positions, row_tables, 0, allocations, active, active & endpoints + ) + prelude = self._prelude(tokens, prelude_kv) + hidden = torch.where(fresh[:, None], prelude, retained) for offset in range(self.config.draft_loops): depth = torch.where(active, depths + offset, 0) hidden = self._execute_recurrent_step( hidden, positions, row_tables, depth, allocations, active ) - choices = self._coda(hidden).argmax(-1) + choices = self._coda(hidden, coda_kv).argmax(-1) start = 0 for i, (span, state, w) in enumerate(zip(spans, states, widths)): p = state.pipeline From ae3611e8c50413d23e7c85fe163559b18c4c67ac Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Tue, 17 Nov 2026 23:54:13 -0500 Subject: [PATCH 2/7] feat(dory): replay boundary stages in speculative CUDA Graphs Asynchronous rounds capture bucketed PRELUDE and CODA graphs (DeviceBoundaryGraphs) next to the recurrent graphs. Positions, write addresses, write masks and context lengths are graph inputs. Thus the graphs keep the device-owned progress of the rounds. The engine captures all buckets at construction. Synchronous rounds of a boundary-KV model use the host-planned boundary graphs of ordinary decode (PreludeGraphs, and CodaGraphs with the cache). The previous synchronous CODA graph had no KV input, and Dory cannot run it. The engine accepts CUDA Graphs for boundary-KV self-speculation. Preemption stays rejected. --- tests/test_async_speculative_sampling_plan.py | 4 +- tests/test_dory_speculative.py | 2 +- tests/test_dory_speculative_gpu.py | 79 ++++++++++++++----- vllm_rlt/engine/stage_engine.py | 10 +-- vllm_rlt/models/dory.py | 4 +- vllm_rlt/worker/async_speculative.py | 21 +++-- vllm_rlt/worker/cuda_graph.py | 74 +++++++++++++++++ vllm_rlt/worker/speculative.py | 19 ++++- 8 files changed, 167 insertions(+), 46 deletions(-) diff --git a/tests/test_async_speculative_sampling_plan.py b/tests/test_async_speculative_sampling_plan.py index fd11991a..882d3eda 100644 --- a/tests/test_async_speculative_sampling_plan.py +++ b/tests/test_async_speculative_sampling_plan.py @@ -211,8 +211,10 @@ def __init__(self, monkeypatch, seed): runner.rng = RngRegistry() runner.slot_values = torch.zeros((8, 2), dtype=torch.long) runner.runtime = SimpleNamespace( - forward=lambda batch: SimpleNamespace(hidden=self.E[batch.tokens]) + prepare=lambda batch: batch, + forward=lambda batch: SimpleNamespace(hidden=self.E[batch.tokens]), ) + runner.prelude_graphs = None runner._execute_recurrent_step = lambda hidden, positions, tables, depth, allocations: ( hidden if depth < 1 else hidden @ self.M ) diff --git a/tests/test_dory_speculative.py b/tests/test_dory_speculative.py index be5e4877..c214cfb4 100644 --- a/tests/test_dory_speculative.py +++ b/tests/test_dory_speculative.py @@ -430,7 +430,7 @@ def test_unqualified_speculative_combinations_fail_at_construction(): engine(model, 2, 1, execution_config=ExecutionConfig(async_scheduling=True)) with pytest.raises(ValueError, match="CUDA graphs"): engine(model, 2, 1, execution_config=ExecutionConfig(cuda_graphs=True)) - with pytest.raises(ValueError, match="without CUDA graphs or preemption"): + with pytest.raises(ValueError, match="does not support preemption"): engine( model, 2, diff --git a/tests/test_dory_speculative_gpu.py b/tests/test_dory_speculative_gpu.py index 007a3ea6..e3df0641 100644 --- a/tests/test_dory_speculative_gpu.py +++ b/tests/test_dory_speculative_gpu.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 -"""Asynchronous Dory self-speculation on CUDA: block and token-pipeline rounds. +"""Dory self-speculation on CUDA: asynchronous rounds and CUDA Graphs. Every case needs an explicit ``CUDA_VISIBLE_DEVICES`` and ``--run-gpu``. Each -mode must serve the greedy tokens of the dense oracle. Forced drafts check that -the committed KV of every plane is the serial target history while the next -round is already submitted. +execution mode must serve the greedy tokens of the dense oracle. Forced drafts +check that the committed KV of every plane is the serial target history while +the next round is already submitted. """ import pytest @@ -28,6 +28,11 @@ # Prompts shorter than and longer than the tiny window of four. PROMPTS = {"a": [4, 7, 3], "b": [9], "c": [5, 6, 8, 2, 11]} LIMITS = {"a": 12, "b": 1, "c": 7} +EXECUTION = { + "async_eager": dict(asynchronous=True, graphs=False), + "async_graph": dict(asynchronous=True, graphs=True), + "sync_graph": dict(asynchronous=False, graphs=True), +} def reference_pair(loops, **overrides): @@ -39,13 +44,13 @@ def reference_pair(loops, **overrides): return reference, model.to("cuda").eval() -def engine(model, k, draft, mode="block", *, asynchronous=True): +def engine(model, k, draft, mode="block", *, asynchronous=True, graphs=False): return LLMEngine( model, attention_backend="triton", cache_config=CacheConfig(num_blocks=1024, block_size=2), scheduler_config=SchedulerConfig(max_num_seqs=4), - execution_config=ExecutionConfig(async_scheduling=asynchronous), + execution_config=ExecutionConfig(async_scheduling=asynchronous, cuda_graphs=graphs), speculative_config=SpeculativeConfig( k, draft_loops=draft, target_loops=model.config.total_ut_steps, mode=mode ), @@ -84,6 +89,12 @@ def assert_released(e): assert e.cache_manager.num_used_blocks == 0 +def assert_graphs_replayed(e): + runner = e.speculative_runner + for graphs in (runner.prelude_graphs, runner.graphs, runner.coda_graphs): + assert graphs.replays > 0 + + def serial_cache(reference, tokens): """KV of the reference schedule: one token per step at the full depth.""" cache = make_cache(reference) @@ -171,12 +182,13 @@ def controlled(hidden, boundary=None): monkeypatch.setattr(runner, "_coda", controlled) +@pytest.mark.parametrize("execution", EXECUTION) @pytest.mark.parametrize("mode", ["block", "pipeline"]) @pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) @pytest.mark.parametrize("k", [1, 3]) -def test_async_greedy_matches_the_dense_reference(mode, draft, target, k): +def test_greedy_matches_the_dense_reference(execution, mode, draft, target, k): reference, model = reference_pair(target) - e = engine(model, k, draft, mode) + e = engine(model, k, draft, mode, **EXECUTION[execution]) for rid, prompt in PROMPTS.items(): e.add_request(rid, prompt, params(model, LIMITS[rid])) final = drain(e) @@ -184,7 +196,18 @@ def test_async_greedy_matches_the_dense_reference(mode, draft, target, k): expected = greedy_reference(reference, prompt, LIMITS[rid])[len(prompt) :] assert final[rid].token_ids == expected assert final[rid].exit_depths == [target] * LIMITS[rid] - assert e.speculative_runner.submitted_before_collect > 0 + runner = e.speculative_runner + if EXECUTION[execution]["graphs"]: + assert_graphs_replayed(e) + if not EXECUTION[execution]["asynchronous"]: + assert not runner.pipeline_states + assert e.cache_manager.num_used_blocks == 0 + return + assert runner.submitted_before_collect > 0 + if EXECUTION[execution]["graphs"]: + # Every async row count fits a captured bucket. + for graphs in (runner.prelude_graphs, runner.graphs, runner.coda_graphs): + assert graphs.fallbacks == 0 assert_released(e) @@ -197,10 +220,11 @@ def bootstrap(e, model, prompt, count): pytest.fail("bootstrap did not produce a token") +@pytest.mark.parametrize("graphs", [False, True]) @pytest.mark.parametrize("mode", ["block", "pipeline"]) -def test_submission_reads_no_device_value(mode, monkeypatch): +def test_submission_reads_no_device_value(mode, graphs, monkeypatch): _, model = reference_pair(4) - e = engine(model, 3, 1, mode) + e = engine(model, 3, 1, mode, graphs=graphs) request = bootstrap(e, model, [4, 7, 3], 16) def forbidden(*args, **kwargs): @@ -218,14 +242,17 @@ def forbidden(*args, **kwargs): assert_released(e) +@pytest.mark.parametrize("graphs", [False, True]) @pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) @pytest.mark.parametrize("rejected", [0, 1, 2, 3]) -def test_block_commits_target_history_through_each_rejection(draft, target, rejected, monkeypatch): +def test_block_commits_target_history_through_each_rejection( + draft, target, rejected, graphs, monkeypatch +): """rejected=3 accepts all three candidates and commits the bonus token.""" reference, model = reference_pair(target) prompt, count = [4, 7, 3], 13 sequence = greedy_reference(reference, prompt, count) - e = engine(model, 3, draft) + e = engine(model, 3, draft, graphs=graphs) force_drafts(monkeypatch, e, sequence, lambda position, index: index == rejected) e.add_request("r", prompt, params(model, count)) request = e.scheduler.requests["r"] @@ -248,13 +275,16 @@ def test_block_commits_target_history_through_each_rejection(draft, target, reje else: assert stats.accepted_tokens == 0 assert request.generated_token_ids == sequence[len(prompt) :] + if graphs: + assert_graphs_replayed(e) assert_released(e) +@pytest.mark.parametrize("graphs", [False, True]) @pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) @pytest.mark.parametrize("pattern", ["all", "none", "every_third"]) def test_pipeline_commits_target_history_under_forced_acceptance( - draft, target, pattern, monkeypatch + draft, target, pattern, graphs, monkeypatch ): reference, model = reference_pair(target) prompt, count = [4, 7, 3], 12 @@ -264,7 +294,7 @@ def test_pipeline_commits_target_history_under_forced_acceptance( "none": lambda position, index: True, "every_third": lambda position, index: position % 3 == 0, }[pattern] - e = engine(model, 3, draft, "pipeline") + e = engine(model, 3, draft, "pipeline", graphs=graphs) force_drafts(monkeypatch, e, sequence, wrong) e.add_request("r", prompt, params(model, count)) request = e.scheduler.requests["r"] @@ -287,6 +317,8 @@ def test_pipeline_commits_target_history_under_forced_acceptance( assert stats.accepted_tokens == 0 else: assert 0 < stats.accepted_tokens < verified + if graphs: + assert_graphs_replayed(e) assert_released(e) @@ -300,8 +332,9 @@ def stage_kv(e, layers, position): ) +@pytest.mark.parametrize("graphs", [False, True]) @pytest.mark.parametrize(("draft", "target"), DEPTH_PAIRS) -def test_pipeline_keeps_the_boundary_kv_of_rows_in_flight(draft, target, monkeypatch): +def test_pipeline_keeps_the_boundary_kv_of_rows_in_flight(draft, target, graphs, monkeypatch): """A pipeline step stores boundary KV only for the rows that use that stage. A retained row keeps the input-stage KV of its first pass. A row between @@ -309,7 +342,7 @@ def test_pipeline_keeps_the_boundary_kv_of_rows_in_flight(draft, target, monkeyp which the draft rows behind it attend. """ _, model = reference_pair(target, ngpt_alpha_init=1.0) - e = engine(model, 3, draft, "pipeline") + e = engine(model, 3, draft, "pipeline", graphs=graphs) runner = e.speculative_runner coda = runner._coda steps = [] @@ -346,13 +379,16 @@ def recording(hidden, boundary=None): between += 1 assert retained assert bool(between) == (target - draft > draft) + if graphs: + assert_graphs_replayed(e) assert_released(e) +@pytest.mark.parametrize("graphs", [False, True]) @pytest.mark.parametrize("mode", ["block", "pipeline"]) -def test_cancellation_with_rounds_in_flight_allows_id_reuse(mode): +def test_cancellation_with_rounds_in_flight_allows_id_reuse(mode, graphs): reference, model = reference_pair(4) - e = engine(model, 3, 2, mode) + e = engine(model, 3, 2, mode, graphs=graphs) e.add_request("a", [4, 7, 3], params(model, 14)) e.add_request("b", [9, 5], params(model, 8)) final = {} @@ -369,10 +405,11 @@ def test_cancellation_with_rounds_in_flight_allows_id_reuse(mode): assert_released(e) +@pytest.mark.parametrize("graphs", [False, True]) @pytest.mark.parametrize("mode", ["block", "pipeline"]) -def test_failed_round_releases_requests_and_allows_id_reuse(mode, monkeypatch): +def test_failed_round_releases_requests_and_allows_id_reuse(mode, graphs, monkeypatch): reference, model = reference_pair(2) - e = engine(model, 3, 1, mode) + e = engine(model, 3, 1, mode, graphs=graphs) bootstrap(e, model, [4, 7, 3], 8) runner = e.speculative_runner name = "_pipeline_step" if mode == "pipeline" else "_verify" diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 5473a564..696e72dd 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -99,13 +99,9 @@ def __init__( if speculative_config is not None: if not getattr(model, "supports_self_speculation", False): raise ValueError("boundary-KV speculative decoding is not yet supported") - # Qualified: synchronous and asynchronous eager rounds. Graph replay and - # resumable preemption of draft/target boundary state are separate work. - if self.execution_config.cuda_graphs or scheduler_config.enable_preemption: - raise ValueError( - "boundary-KV speculative decoding requires eager execution " - "without CUDA graphs or preemption" - ) + # Resumable preemption of draft/target boundary state is separate work. + if scheduler_config.enable_preemption: + raise ValueError("boundary-KV speculative decoding does not support preemption") if not getattr(model, "supports_cuda_graphs", True) and self.execution_config.cuda_graphs: raise ValueError("this model does not support CUDA graphs yet") self.speculative_config = speculative_config diff --git a/vllm_rlt/models/dory.py b/vllm_rlt/models/dory.py index 8e6a388e..a26e1145 100644 --- a/vllm_rlt/models/dory.py +++ b/vllm_rlt/models/dory.py @@ -488,8 +488,8 @@ class DoryForCausalLM(nn.Module): supports_async_scheduling = True # Selection must reject a backend that would silently attend full history. requires_sliding_window = True - # Synchronous self-speculation with per-loop KV: a draft's loops are the - # target's first loops, and boundary KV is committed by the target pass. + # Self-speculation with per-loop KV, in every execution mode: a draft's loops + # are the target's first loops, and boundary KV is committed by the target pass. supports_self_speculation = True def __init__(self, config: DoryConfig) -> None: diff --git a/vllm_rlt/worker/async_speculative.py b/vllm_rlt/worker/async_speculative.py index decff876..a42f9e8b 100644 --- a/vllm_rlt/worker/async_speculative.py +++ b/vllm_rlt/worker/async_speculative.py @@ -25,6 +25,7 @@ from vllm_rlt.request import Operation, Request from vllm_rlt.worker.cuda_graph import ( CodaGraphs, + DeviceBoundaryGraphs, DeviceRecurrentGraphs, speculative_capture_sizes, ) @@ -368,19 +369,17 @@ def __init__( compute_stream=compute_stream, ) sizes = speculative_capture_sizes(execution, scheduler, config) - self.graphs = ( - DeviceRecurrentGraphs(model, cache, execution, False, capture_sizes=sizes) - if execution.cuda_graphs - else None - ) - self.coda_graphs = ( - CodaGraphs(model, execution, bucketed=True, capture_sizes=sizes) - if execution.cuda_graphs - else None - ) - if self.graphs is not None: + if execution.cuda_graphs: + self.graphs = DeviceRecurrentGraphs(model, cache, execution, False, capture_sizes=sizes) + if self.boundary_kv: + self.prelude_graphs = DeviceBoundaryGraphs(model, cache, Operation.PRELUDE, sizes) + self.coda_graphs = DeviceBoundaryGraphs(model, cache, Operation.CODA, sizes) + else: + self.coda_graphs = CodaGraphs(model, execution, bucketed=True, capture_sizes=sizes) self.graphs.prepare() self.coda_graphs.prepare() + if self.prelude_graphs is not None: + self.prelude_graphs.prepare() self.compute_stream = compute_stream self.copy_stream = copy_stream self.compute_stream.wait_stream(torch.cuda.current_stream(self.device)) diff --git a/vllm_rlt/worker/cuda_graph.py b/vllm_rlt/worker/cuda_graph.py index c0f0ceac..053ca9d7 100644 --- a/vllm_rlt/worker/cuda_graph.py +++ b/vllm_rlt/worker/cuda_graph.py @@ -9,6 +9,7 @@ import torch from vllm_rlt.config import cuda_graph_rows +from vllm_rlt.request import Operation from vllm_rlt.worker.metadata import DeviceKVMetadata logger = logging.getLogger(__name__) @@ -534,3 +535,76 @@ def _run(self, entry): def _eager(self, tokens, batch): return self.model.prelude_prepared(tokens, batch, self.cache) + + +class DeviceBoundaryGraphs: + """Bucketed input- or output-stage graphs of a boundary-KV model. + + Asynchronous speculation keeps row positions on the device, so positions, + write addresses, write masks and context lengths stay graph inputs. Entries + hold no request allocation and never touch CPU written-position state. + """ + + def __init__(self, model, cache, operation, capture_sizes): + self.cache = cache + self.capture_sizes = capture_sizes + self.max_positions = model.config.max_position_embeddings + if operation == Operation.PRELUDE: + self.stage = model.prelude_prepared + self.example = torch.empty(0, dtype=torch.long, device=cache.device) + else: + self.stage = model.coda_prepared + self.example = next(model.parameters()).new_empty((0, model.config.hidden_size)) + self.entries = {} + self.pool = torch.cuda.graph_pool_handle() + self.stream = torch.cuda.Stream(device=cache.device) + self.captures = self.replays = self.fallbacks = 0 + + @torch.inference_mode() + def prepare(self): + """Capture every bucket before any serving request exists.""" + device = self.cache.device + batch = SimpleNamespace( + position_ids=torch.empty(0, device=device, dtype=torch.long), + write_blocks=torch.empty(0, device=device, dtype=torch.long), + write_offsets=torch.empty(0, device=device, dtype=torch.long), + context_lengths=torch.empty(0, device=device, dtype=torch.int32), + block_tables=torch.empty((0, 1), device=device, dtype=torch.int32), + cu_seqlens_q=None, + ) + proxy = _DeviceCache(self.cache) + for size in self.capture_sizes: + entry = _graph_entry( + self.example, batch, self.cache, self.max_positions, count=size, bucketed=True + ) + _copy_graph_inputs(entry, self.example, batch, count=0) + entry.graph, entry.output = _capture( + entry.hidden, + self.stream, + self.pool, + lambda: self.stage(entry.hidden, entry.metadata, proxy), + startup=True, + ) + self.entries[size] = entry + self.captures += 1 + + def select(self, inputs, batch): + count = _query_count(batch) + if count > self.capture_sizes[-1]: + self.fallbacks += 1 + return PreparedGraph(batch) + return PreparedGraph(batch, self.entries[_bucket_size(self.capture_sizes, count)]) + + @torch.inference_mode() + def prepare_inputs(self, call, inputs): + if call.entry is not None: + _copy_graph_inputs(call.entry, inputs, call.metadata) + + @torch.inference_mode() + def execute(self, call, inputs): + if call.entry is None: + return self.stage(inputs, call.metadata, self.cache) + call.entry.graph.replay() + self.replays += 1 + # The next replay of this bucket overwrites the static output. + return call.entry.output[: _query_count(call.metadata)].clone() diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index d292e422..9a1ee2d1 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -20,7 +20,7 @@ from vllm_rlt.core.scheduler import SchedulerOutput from vllm_rlt.request import Operation, Request from vllm_rlt.worker.buffers import stage_metadata -from vllm_rlt.worker.cuda_graph import CodaGraphs, RecurrentGraphs +from vllm_rlt.worker.cuda_graph import CodaGraphs, PreludeGraphs, RecurrentGraphs from vllm_rlt.worker.executor import ForwardBatch, ModelRuntime from vllm_rlt.worker.metadata import KVMetadata from vllm_rlt.worker.output import ModelRunnerOutput, SpeculativeResult @@ -216,7 +216,13 @@ def __init__( self.graphs = ( RecurrentGraphs(model, cache, execution, False) if execution.cuda_graphs else None ) - self.coda_graphs = CodaGraphs(model, execution) if execution.cuda_graphs else None + self.coda_graphs = self.prelude_graphs = None + if execution.cuda_graphs and self.boundary_kv: + # Boundary stages attend KV, so their graphs take KV metadata as input. + self.coda_graphs = CodaGraphs(model, execution, cache) + self.prelude_graphs = PreludeGraphs(model, execution, cache) + elif execution.cuda_graphs: + self.coda_graphs = CodaGraphs(model, execution) def _packed(self, packed): capabilities = self.cache.attention_capabilities @@ -315,7 +321,14 @@ def _prepare_inputs(self, batch): def _prelude(self, tokens, boundary=None): return self.runtime.forward( - ForwardBatch(Operation.PRELUDE, tokens=tokens, prelude_coda_kv_metadata=boundary) + self.runtime.prepare( + ForwardBatch( + Operation.PRELUDE, + tokens=tokens, + prelude_coda_kv_metadata=boundary, + graphs=self.prelude_graphs, + ) + ) ).hidden def _recurrent(self, hidden, metadata): From f2c3623112dcec669bec1dfdffdafffce11520af Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Tue, 17 Nov 2026 23:54:17 -0500 Subject: [PATCH 3/7] perf(speculative): update asynchronous pipeline requests in one batch The asynchronous pipeline step updated the device state of each request with its own small operations, about 35 kernel launches for each request in each step. At 64 requests, these launches made asynchronous eager rounds 27 % slower than synchronous eager rounds. They also ran outside the CUDA Graphs. The pipeline state of all live requests is now one pool, indexed by the slot table that block rounds already use. A step gathers the rows of its requests, updates them in one batch and scatters them back. Thus the number of operations does not change with the request count. The rows, their order and the results do not change. Preemption snapshots and restoration use the pool slot of the request. --- vllm_rlt/worker/async_speculative.py | 309 +++++++++++++++------------ 1 file changed, 178 insertions(+), 131 deletions(-) diff --git a/vllm_rlt/worker/async_speculative.py b/vllm_rlt/worker/async_speculative.py index a42f9e8b..3a06e42a 100644 --- a/vllm_rlt/worker/async_speculative.py +++ b/vllm_rlt/worker/async_speculative.py @@ -206,7 +206,11 @@ def merge_random_round( @dataclass class DevicePipelineState: - """Fixed-capacity, oldest-first token slots; all validity and depths live on GPU.""" + """Fixed-capacity, oldest-first token slots; all validity and depths live on GPU. + + The runner keeps one pool with a leading slot dimension. A request state holds + the views of its own slot. + """ tokens: torch.Tensor depths: torch.Tensor @@ -214,6 +218,22 @@ class DevicePipelineState: candidates: torch.Tensor active: torch.Tensor + @classmethod + def zeros(cls, slots, width, hidden_size, dtype, device): + return cls( + torch.zeros((slots, width), dtype=torch.long, device=device), + torch.zeros((slots, width), dtype=torch.long, device=device), + torch.zeros((slots, width, hidden_size), dtype=dtype, device=device), + torch.zeros((slots, width), dtype=torch.long, device=device), + torch.zeros((slots, width), dtype=torch.bool, device=device), + ) + + def tensors(self): + return (self.tokens, self.depths, self.hidden, self.candidates, self.active) + + def row(self, slot): + return DevicePipelineState(*(tensor[slot] for tensor in self.tensors())) + @dataclass class SpeculativeRequestState: @@ -392,6 +412,22 @@ def __init__( ) self.free_slots = list(reversed(range(len(self.slot_values)))) self._retired_slot_values: list[torch.Tensor] = [] + # Pipeline token slots of every live request, indexed like the slot table, so + # a step gathers, updates and scatters all its requests in one operation each. + self.pipeline_width = min( + config.num_speculative_tokens + 1, config.target_loops // config.draft_loops + ) + self.pipeline_pool = ( + DevicePipelineState.zeros( + len(self.slot_values), + self.pipeline_width, + model.config.hidden_size, + next(model.parameters()).dtype, + self.device, + ) + if config.mode == "pipeline" + else None + ) self.submitted_before_collect = 0 self.discarded_rounds = 0 self.readback_waits = 0 @@ -408,36 +444,29 @@ def _state(self, request, buffer): host = torch.tensor([request.input_token_id, request.position], pin_memory=True) values = host.to(self.device, non_blocking=True) buffer.keepalive.extend((host, values)) - if self.config.mode == "pipeline": - state = SpeculativeRequestState(request, allocation, values[0], values[1]) - else: - if not self.free_slots: - self._grow_slots() - slot = self.free_slots.pop() - self.slot_values[slot].copy_(values) - state = SpeculativeRequestState( - request, allocation, self.slot_values[slot, 0], self.slot_values[slot, 1], slot=slot - ) - if self.config.mode == "pipeline": - width = min( - self.config.num_speculative_tokens + 1, - self.config.target_loops // self.config.draft_loops, - ) - parameter = next(self.model.parameters()) - state.pipeline = DevicePipelineState( - state.token.expand(width).clone(), - torch.zeros(width, dtype=torch.long, device=self.device), - torch.zeros( - (width, self.model.config.hidden_size), - dtype=parameter.dtype, - device=self.device, - ), - torch.zeros(width, dtype=torch.long, device=self.device), - torch.arange(width, device=self.device) == 0, - ) + if not self.free_slots: + self._grow_slots() + slot = self.free_slots.pop() + self.slot_values[slot].copy_(values) + if self.pipeline_pool is not None: + pool = self.pipeline_pool + pool.tokens[slot].copy_(values[0].expand(self.pipeline_width)) + for tensor in (pool.depths, pool.hidden, pool.candidates, pool.active): + tensor[slot].zero_() + pool.active[slot, 0] = True + state = SpeculativeRequestState( + request, allocation, *self.slot_values[slot].unbind(), slot=slot + ) + if self.pipeline_pool is not None: + state.pipeline = self.pipeline_pool.row(slot) self.states[rid] = state return state + def bind_slot(self, state): + state.token, state.position = self.slot_values[state.slot].unbind() + if self.pipeline_pool is not None: + state.pipeline = self.pipeline_pool.row(state.slot) + def _grow_slots(self): """Double the slot table on the compute stream; live states move with it.""" old = self.slot_values @@ -446,9 +475,18 @@ def _grow_slots(self): # Work already queued still addresses the old storage. self._retired_slot_values.append(old) self.slot_values = new + if self.pipeline_pool is not None: + pool = self.pipeline_pool + grown = DevicePipelineState.zeros( + len(new), self.pipeline_width, pool.hidden.shape[2], pool.hidden.dtype, self.device + ) + for source, target in zip(pool.tensors(), grown.tensors()): + target[: len(old)].copy_(source) + self._retired_slot_values.extend(pool.tensors()) + self.pipeline_pool = grown self.free_slots = list(reversed(range(len(old), len(new)))) + self.free_slots for state in self.states.values(): - state.token, state.position = new[state.slot, 0], new[state.slot, 1] + self.bind_slot(state) def device_metadata(self, positions, tables, depth, allocations, active=None, writes=None): """Describe rows whose positions stay on the device. @@ -852,103 +890,116 @@ def submit(self, batch): ) def _pipeline_step(self, spans, states, tables, allocations, buffer): - widths = [ - min(span.token_count, len(state.pipeline.tokens)) for span, state in zip(spans, states) + pool, width = self.pipeline_pool, self.pipeline_width + n = len(spans) + # Budget reductions discard speculative descendants, never the root. + widths = np.array([min(span.token_count, width) for span in spans], dtype=np.int64) + # Row r of the step is slot column flat[r] % width of request requests[r], + # request-major and oldest first, as the per-request rows were before. + requests = np.repeat(np.arange(n), widths) + columns = np.arange(len(requests)) - np.repeat(np.cumsum(widths) - widths, widths) + limits = [ + len(span.request.prompt_token_ids) + span.request.sampling_params.max_tokens - 1 + for span in spans ] - active = torch.cat([state.pipeline.active[:w] for state, w in zip(states, widths)]) - depths = torch.cat([state.pipeline.depths[:w] for state, w in zip(states, widths)]) - positions = torch.cat( - [ - state.position + torch.arange(w, device=self.device) - for state, w in zip(states, widths) - ] - ) + host = torch.from_numpy( + np.concatenate( + ( + [state.slot for state in states], + widths, + limits, + requests, + requests * width + columns, + ) + ) + ).pin_memory() + values = host.to(self.device, non_blocking=True) + buffer.keepalive.extend((host, values)) + slots, w, limit, row_requests, flat = values.split((n, n, n, len(requests), len(requests))) + tokens, depths, retained, candidates, active = (t[slots] for t in pool.tensors()) + token, position = self.slot_values[slots].unbind(1) + ids = torch.arange(width, device=self.device) + row_active = active.view(-1)[flat] + row_depths = depths.view(-1)[flat] # Padding never reads beyond the reserved context or writes live KV. - positions = torch.where(active, positions, 0) - tokens = torch.cat([state.pipeline.tokens[:w] for state, w in zip(states, widths)]) - retained = torch.cat([state.pipeline.hidden[:w] for state, w in zip(states, widths)]) - row_tables = torch.cat([tables[i : i + 1].expand(w, -1, -1) for i, w in enumerate(widths)]) - fresh = depths == 0 + positions = torch.where(row_active, (position[:, None] + ids).view(-1)[flat], 0) + row_tables = tables[row_requests] + fresh = row_depths == 0 prelude_kv = coda_kv = None if self.boundary_kv: # Every row runs both boundary stages to keep one call shape. A # retained row already holds the input-stage KV of its token. prelude_kv = self.device_metadata( - positions, row_tables, 0, allocations, active, active & fresh + positions, row_tables, 0, allocations, row_active, row_active & fresh ) # Only rows that reach the draft or target depth read a distribution. # The output-stage KV of other rows must stay the KV of their draft pass. - reached = depths + self.config.draft_loops + reached = row_depths + self.config.draft_loops endpoints = (reached == self.config.draft_loops) | (reached == self.config.target_loops) coda_kv = self.device_metadata( - positions, row_tables, 0, allocations, active, active & endpoints + positions, row_tables, 0, allocations, row_active, row_active & endpoints ) - prelude = self._prelude(tokens, prelude_kv) - hidden = torch.where(fresh[:, None], prelude, retained) + prelude = self._prelude(tokens.view(-1)[flat], prelude_kv) + hidden = torch.where(fresh[:, None], prelude, retained.view(n * width, -1)[flat]) for offset in range(self.config.draft_loops): - depth = torch.where(active, depths + offset, 0) + depth = torch.where(row_active, row_depths + offset, 0) hidden = self._execute_recurrent_step( - hidden, positions, row_tables, depth, allocations, active + hidden, positions, row_tables, depth, allocations, row_active ) choices = self._coda(hidden, coda_kv).argmax(-1) - start = 0 - for i, (span, state, w) in enumerate(zip(spans, states, widths)): - p = state.pipeline - ids = torch.arange(len(p.tokens), device=self.device) - # Budget reductions discard speculative descendants, never the root. - valid = p.active & (ids < w) - new_depths = p.depths + valid.long() * self.config.draft_loops - predictions = torch.zeros_like(p.tokens) - predictions[:w].copy_(choices[start : start + w]) - next_hidden = p.hidden.clone() - next_hidden[:w].copy_(hidden[start : start + w]) - drafted = valid & (new_depths == self.config.draft_loops) - candidates = torch.where(drafted, predictions, p.candidates) - emitted = valid[0] & (new_depths[0] == self.config.target_loops) - accepted = emitted & (candidates[0] == predictions[0]) - reject = emitted & ~accepted - state.token.copy_(torch.where(emitted, predictions[0], state.token)) - state.position.add_(emitted.long()) - # Shift only on target completion. All decisions remain stream-ordered tensors. - source = (ids + emitted.long()).clamp_max(len(ids) - 1) - remaining = valid.long().sum() - emitted.long() - surviving = (ids < remaining) & ~reject - p.tokens.copy_(p.tokens.gather(0, source)) - p.depths.copy_(new_depths.gather(0, source)) - p.hidden.copy_(next_hidden[source]) - p.candidates.copy_(candidates.gather(0, source)) - tail = (valid.long().sum() - 1).clamp_min(0) - append = drafted.gather(0, tail.reshape(1)).squeeze(0) & ~reject - newborn = (ids == remaining) & append - # A rejection (or a one-row budget) restarts from the corrected root. - restart = reject | (emitted & (remaining == 0)) - newborn = torch.where(restart, ids == 0, newborn) - limit = len(span.request.prompt_token_ids) + span.request.sampling_params.max_tokens - 1 - newborn &= (ids < w) & (state.position + ids < limit) - candidate = candidates.gather(0, tail.reshape(1)).squeeze(0) - token = torch.where(restart, state.token, candidate) - p.tokens.copy_(torch.where(newborn, token, p.tokens)) - p.depths.masked_fill_(newborn, 0) - p.active.copy_((surviving | newborn) & (state.position + ids < limit)) - # Per-request wire order matches SpeculativeRoundTicket.collect(): - # prediction, accepted, emitted, next_position, drafted, verified. - # A pipeline step verifies at most one token per request, so the - # verified count equals emitted (0 or 1). - buffer.result[6 * i : 6 * i + 6].copy_( - torch.stack( - ( - predictions[0], - accepted.long(), - emitted.long(), - state.position, - drafted.long().sum(), - emitted.long(), - ) - ) + # One update for all requests; every decision stays a stream-ordered tensor. + valid = active & (ids < w[:, None]) + new_depths = depths + valid.long() * self.config.draft_loops + predictions = torch.zeros_like(tokens) + predictions.view(-1)[flat] = choices + retained.view(n * width, -1)[flat] = hidden + drafted = valid & (new_depths == self.config.draft_loops) + candidates = torch.where(drafted, predictions, candidates) + emitted = valid[:, 0] & (new_depths[:, 0] == self.config.target_loops) + accepted = emitted & (candidates[:, 0] == predictions[:, 0]) + reject = emitted & ~accepted + token = torch.where(emitted, predictions[:, 0], token) + position = position + emitted.long() + # Shift only on target completion. + source = (ids + emitted.long()[:, None]).clamp_max(width - 1) + count = valid.long().sum(1) + remaining = count - emitted.long() + surviving = (ids < remaining[:, None]) & ~reject[:, None] + tail = (count - 1).clamp_min(0)[:, None] + append = drafted.gather(1, tail).squeeze(1) & ~reject + newborn = (ids == remaining[:, None]) & append[:, None] + # A rejection (or a one-row budget) restarts from the corrected root. + restart = reject | (emitted & (remaining == 0)) + newborn = torch.where(restart[:, None], ids == 0, newborn) + inside = position[:, None] + ids < limit[:, None] + newborn &= (ids < w[:, None]) & inside + start = torch.where(restart, token, candidates.gather(1, tail).squeeze(1)) + pool.tokens[slots] = torch.where(newborn, start[:, None], tokens.gather(1, source)) + pool.depths[slots] = new_depths.gather(1, source).masked_fill(newborn, 0) + pool.hidden[slots] = retained.gather(1, source[:, :, None].expand_as(retained)) + pool.candidates[slots] = candidates.gather(1, source) + pool.active[slots] = (surviving | newborn) & inside + self.slot_values[slots] = torch.stack((token, position), 1) + # Per-request wire order matches SpeculativeRoundTicket.collect(): + # prediction, accepted, emitted, next_position, drafted, verified. + # A pipeline step verifies at most one token per request, so the + # verified count equals emitted (0 or 1). + buffer.result[: 6 * n].view(n, 6).copy_( + torch.stack( + ( + predictions[:, 0], + accepted.long(), + emitted.long(), + position, + drafted.long().sum(1), + emitted.long(), + ), + 1, ) + ) + for state in states: state.submitted += 1 - start += w - return start + return len(requests) def snapshot_pipeline(self, request, device): state = self.states.get(request.request_id) @@ -988,44 +1039,40 @@ def restore_pipeline(self, request, snapshot): raise RuntimeError("invalid pipeline restoration generation") if snapshot.pending or snapshot.position != request.position: raise RuntimeError("pipeline snapshot disagrees with committed request position") - width = min( - self.config.num_speculative_tokens + 1, - self.config.target_loops // self.config.draft_loops, - ) + width = self.pipeline_width if len(snapshot.tokens) > width: raise RuntimeError("pipeline snapshot exceeds the configured window") with torch.cuda.stream(self.compute_stream): + if not self.free_slots: + self._grow_slots() + slot = self.free_slots.pop() + pool = self.pipeline_pool + padding = [0] * (width - len(snapshot.tokens)) values = torch.tensor( [ - [t.token for t in snapshot.tokens] + [0] * (width - len(snapshot.tokens)), - [t.depth for t in snapshot.tokens] + [0] * (width - len(snapshot.tokens)), - [t.candidate or 0 for t in snapshot.tokens] - + [0] * (width - len(snapshot.tokens)), + [t.token for t in snapshot.tokens] + padding, + [t.depth for t in snapshot.tokens] + padding, + [t.candidate or 0 for t in snapshot.tokens] + padding, ], device=self.device, ) - hidden = torch.zeros( - (width, self.model.config.hidden_size), - device=self.device, - dtype=next(self.model.parameters()).dtype, - ) + pool.tokens[slot], pool.depths[slot], pool.candidates[slot] = values + pool.hidden[slot].zero_() for i, token in enumerate(snapshot.tokens): if token.hidden is not None: - hidden[i].copy_(token.hidden) - pipeline = DevicePipelineState( - values[0], - values[1], - hidden, - values[2], - torch.arange(width, device=self.device) < len(snapshot.tokens), + pool.hidden[slot, i].copy_(token.hidden) + pool.active[slot] = torch.arange(width, device=self.device) < len(snapshot.tokens) + self.slot_values[slot] = torch.tensor( + [request.input_token_id, snapshot.position], device=self.device ) - self.states[request.request_id] = SpeculativeRequestState( + state = SpeculativeRequestState( request, self.cache._get_allocation(request.request_id), - torch.tensor(request.input_token_id, device=self.device), - torch.tensor(snapshot.position, device=self.device), - pipeline, + *self.slot_values[slot].unbind(), + pool.row(slot), + slot=slot, ) + self.states[request.request_id] = state def request_ready(self, request): state = self.states.get(request.request_id) From a6e2fe2d813ceea4b67baa7cd8c09f920f89a155 Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Tue, 17 Nov 2026 23:55:31 -0500 Subject: [PATCH 4/7] docs(dory): describe asynchronous and graphed self-speculation Describe how asynchronous rounds give PRELUDE and CODA their depth-0 KV, the pipeline write masks, the sampling limit, the shared pipeline slot pool and the graph families of each mode. Add the validation status: the CUDA test file, the real-checkpoint FP32 token agreement at 1 to 2, 1 to 4 and 2 to 4 loops, GSM8K accuracy, and BF16 wall time, first-token time and time per output token on one H100. --- docs/async-speculative.md | 22 ++++- docs/dory.md | 181 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 193 insertions(+), 10 deletions(-) diff --git a/docs/async-speculative.md b/docs/async-speculative.md index 6a003d1a..b03a42a6 100644 --- a/docs/async-speculative.md +++ b/docs/async-speculative.md @@ -983,7 +983,11 @@ combination with compatible attention backends such as Triton and FA4. The asynchronous pipeline uses the same compute/copy streams and two leased result buffers as asynchronous block speculation. The compute stream advances or rejects token slots; the copy stream transfers compact results to pinned host storage. -Collecting an older step never rewinds GPU state. Inactive slots have masked KV +Collecting an older step never rewinds GPU state. The token slots of all live +requests share one device pool, indexed by the slot table of block rounds. A step +gathers the slots of its requests, updates them in one batch and scatters them +back. Thus its number of launched operations does not grow with the number of +requests. Inactive slots have masked KV writes and zero attention lengths. Fixed-shape prelude/coda calls can include unused rows; this avoids a host dependency for selecting dynamic batch sizes. Pending steps cannot be preempted. Waiting admissions drain outstanding steps so @@ -1074,6 +1078,22 @@ The saved work did not offset selection and result-copy overhead. The experiment was reverted; the runtime retains the simpler fixed-slot boundary calls. These results do not rule out a different implementation or workload benefiting. +### Boundary-KV models + +Dory's PRELUDE and CODA contain attention and keep their KV at depth 0. Its +asynchronous rounds keep the fixed-slot boundary calls above. Each call gets +depth-0 device metadata, and a pipeline step adds a write mask: + +- PRELUDE stores KV only for rows at depth 0. A retained row keeps the KV of + its first pass. +- CODA stores KV only for rows that reach the draft or the target depth. A row + between them keeps the KV of its draft pass, which later draft rows attend. + +With CUDA Graphs, `DeviceBoundaryGraphs` replays both stages in the same buckets +as the recurrent loops, and the write mask is a graph input. Asynchronous Dory +rounds take greedy requests only. `docs/dory.md` describes the KV contract and +its validation. + ## Validation The following commands and historical evidence concern the cross-round block diff --git a/docs/dory.md b/docs/dory.md index 46cfd8d6..73d42da3 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -86,6 +86,9 @@ layer id rather than their physical sublayer index. the next input stage and only its readback is deferred. The two switches are independent: sync eager, sync Graph, async eager and async Graph are tested separately. +- Self-speculation in block and token-pipeline mode, in the same four execution + modes. Asynchronous rounds take greedy requests only. See + [Self-speculation](#self-speculation). - The official tokenizer, including the Hub's `tokenizer.json` and chat template, resolved without requiring transformers to accept the checkpoint config (its generic validator rejects Dory's `mlp` sublayer entries). @@ -136,9 +139,9 @@ These fail fast rather than degrade silently: - WCPB, mixed or staged prefill, and PD. - Prefix caching. -- Self-speculation combined with CUDA Graphs, asynchronous scheduling, - preemption or `last_loop` KV, and sampled token-pipeline speculation. The - synchronous eager form is supported; see below. +- Self-speculation combined with preemption or `last_loop` KV, sampled + token-pipeline speculation, and sampled asynchronous rounds. The other forms + are supported; see below. - Per-request loop counts other than the configured full depth: Dory has no trained exit gate, so the runtime refuses to invent one, including via `min_loops` with `exit_threshold=0`. @@ -155,13 +158,15 @@ construction instead of attending full history. Masking accepts host-planned, graph-private and device-owned metadata; storage reclamation of out-of-window pages stays with 2B. -## Synchronous self-speculation +## Self-speculation `SpeculativeConfig(K, draft_loops=d, target_loops=T, mode=...)` drafts with the first `d` recurrent loops and verifies with all `T`, where `T` is the loop count -the model was loaded with. It needs `per_loop` KV, synchronous eager execution -and fixed-depth requests (`min_loops == max_loops == T`). Qualified depth pairs -are 1→2, 1→4 and 2→4. +the model was loaded with. It needs `per_loop` KV and fixed-depth requests +(`min_loops == max_loops == T`). Qualified depth pairs are 1→2, 1→4 and 2→4. +This section describes synchronous eager rounds first; +[asynchronous rounds and CUDA Graphs](#asynchronous-rounds-and-cuda-graphs) keep +the same KV contract. | Mode | Sampling | Round | | --- | --- | --- | @@ -222,6 +227,45 @@ final consumer completes. The pipeline mode uses the same preparation-local layout store and operation indices for its new-token, recurrent and endpoint views; all views are bound before execution. +### Asynchronous rounds and CUDA Graphs + +`ExecutionConfig(async_scheduling=True)` submits the next round before the host +reads the previous one (see `docs/async-speculative.md`). The actual positions +then exist only on the device. Thus the runner builds device metadata +(`DeviceKVMetadata`) at depth 0 for PRELUDE and CODA, as it does for the +recurrent loops. The KV contract does not change: all KV below the start of the +next round is the serial target history. + +- **Block rounds.** Each draft offset gives its rows one depth-0 view for + PRELUDE and the draft CODA. A partial draft CODA gets its own view of the + drafting rows. Verification gives every row one depth-0 view for the target + CODA. +- **Pipeline steps.** Every row runs both boundary stages, so each call keeps one + shape (the fixed-slot calls of `docs/async-speculative.md`). Write masks select + the rows that store KV. PRELUDE stores only for rows at depth 0. CODA stores + only for rows that reach the draft or the target depth. A row between these + depths (1→4) keeps the CODA KV of its draft pass, as in a synchronous step. +- **Sampling.** Asynchronous rounds take greedy requests only. Asynchronous + block rounds can sample for models without boundary KV, but that combination + is not qualified for Dory. Such a request fails when it is added. + +The asynchronous pipeline keeps all its slots full: when the root reaches the +target depth, the newest draft row gets its successor in the same step. The +synchronous pipeline appends only inside the scheduled span. The two modes +therefore run different step sequences, and their acceptance can differ. Both +commit the target tokens. + +`ExecutionConfig(cuda_graphs=True)`: + +- **Asynchronous rounds** replay bucketed graphs for PRELUDE, every recurrent + loop and CODA (`DeviceBoundaryGraphs`, `DeviceRecurrentGraphs`). Positions, + write addresses, write masks and context lengths are graph inputs. The engine + captures all buckets at construction. A row count above the largest bucket + runs eager. +- **Synchronous rounds** use the host-planned boundary graphs of ordinary decode + (`PreludeGraphs`, and `CodaGraphs` with the cache). These capture one graph + per row count, up to `cuda_graph_max_graphs`; other row counts run eager. + ## Validation status `tests/test_dory.py` and `tests/test_dory_gpu.py` hold the regression suites; @@ -390,6 +434,125 @@ Reading the numbers: memory was 74–82 GiB with the automatically sized KV pool; every arm released its KV blocks. +### Asynchronous self-speculation and CUDA Graphs + +`tests/test_dory_speculative_gpu.py` (97 CUDA cases, small FP32 model, Triton) +checks the execution modes against the dense oracle: + +- Greedy tokens for ragged requests in block and pipeline mode, for 1→2, 1→4 and + 2→4 and K = 1 and 3, in async eager, async Graph and sync Graph. +- A forced rejection at every draft position of an async block round, and forced + acceptance patterns of an async pipeline. After every collected round, every + plane below the committed frontier equals the serial target history while the + next round is in flight. +- After every async pipeline step, a retained row keeps its PRELUDE KV and a row + between the draft and target depths keeps the CODA KV of its draft pass, bit + for bit. +- No host read of a device value during submission, cancellation with two + rounds in flight and ID reuse, cleanup after a failed round, and the greedy + and preemption rejections. + +The Graph cases also check that all three graph families replay; async Graph +rounds have no eager fallback. On one H100 (torch 2.15.0+cu132, Triton 3.9), the +file passes together with `tests/test_dory_speculative.py`, +`tests/test_dory_gpu.py`, `tests/test_speculative.py`, `tests/test_cuda_graph.py` +and the `tests/test_async_*` files. The `flash_attn` cases were not selected, +because that package is not installed there. + +Real checkpoint (pinned revision), K = 3, greedy, Triton, one H100, torch +2.15.0+cu132. Prompts are disjoint windows of the WikiText-103 test split. + +**FP32 token agreement.** 32 requests × 512 prompt tokens + 128 generated +tokens. Every speculative arm (sync and async eager pipeline and block, sync +Graph pipeline, async Graph pipeline and block) is compared with synchronous +eager decoding without speculation at the same loop count: + +| Loops | Draft → target | Requests with identical tokens, every arm | Acceptance, pipeline | Acceptance, block | +| --- | --- | --- | --- | --- | +| 2 | 1 → 2 | 32/32 | 0.85 | 0.85 | +| 4 | 1 → 4 | 32/32 | 0.81 sync, 0.73 async | 0.85 | +| 4 | 2 → 4 | 32/32 | 0.93 | 0.92 | + +At 1 → 4 the asynchronous pipeline runs a different step sequence (see +[Asynchronous rounds and CUDA Graphs](#asynchronous-rounds-and-cuda-graphs)), +so its acceptance differs. Its tokens do not. + +**GSM8K, BF16.** The first 200 test questions, 8-shot prompt from the first +eight training examples, at most 256 generated tokens. An answer is correct +when the number after `####` equals the reference. All 200 requests are +submitted at once, at most 64 in flight: + +| Arm | Correct | Wall time (s) | Tokens identical to the baseline | +| --- | --- | --- | --- | +| Baseline: async Graph, no speculation | 161/200 | 188 | — | +| Async Graph pipeline | 164/200 | 78 | 32/200 | +| Async Graph block | 163/200 | 77 | 34/200 | + +BF16 tokens depend on the batch composition, so most sequences differ from the +baseline after some token. Accuracy does not drop. + +**BF16 wall time.** Two loops, a one-loop draft. "Requests" is the number of +concurrent requests, "Prompt" the prompt length of each request in tokens, and +"Generated" the number of tokens that each request generates. All requests are +submitted at once. Each arm runs one warmup and then five timed repetitions in a +row; values are the mean whole-run time in seconds, prefill included. All arms +ran in one container. + +With CUDA Graphs (`cuda_graphs=True` in every column, the baseline included). +Percentages are against the baseline: + +| Requests | Prompt | Generated | Baseline: async Graph, no speculation | Sync Graph pipeline | Async Graph pipeline | Async Graph block | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | 1024 | 128 | 2.48 | 1.80 (−28 %) | **1.77 (−29 %)** | 1.91 (−23 %) | +| 16 | 1024 | 128 | 5.84 | 5.61 (−4 %) | **5.27 (−10 %)** | 5.61 (−4 %) | +| 64 | 937 | 128 | 22.19 | 16.72 (−25 %) | **13.61 (−39 %)** | 14.96 (−33 %) | +| 8 | 7500 | 128 | 20.79 | 20.20 (−3 %) | **20.15 (−3 %)** | 20.36 (−2 %) | + +Time to first token p50 / time per output token p50 / p90, in seconds and +milliseconds: + +| Requests | Prompt | Generated | Baseline | Async Graph pipeline | Async Graph block | +| --- | --- | --- | --- | --- | --- | +| 1 | 1024 | 128 | 0.17 / 18.2 / 18.2 | 0.17 / 12.6 / 12.6 | 0.17 / 13.7 / 13.7 | +| 16 | 1024 | 128 | 1.97 / 35.0 / 40.6 | 1.65 / 28.6 / 35.9 | 1.79 / 30.0 / 36.1 | +| 64 | 937 | 128 | 5.86 / 129.3 / 155.7 | 5.35 / 63.7 / 92.6 | 6.16 / 61.8 / 87.9 | +| 8 | 7500 | 128 | 17.56 / 32.9 / 47.6 | 17.19 / 28.3 / 49.6 | 17.21 / 29.0 / 48.7 | + +Without CUDA Graphs (eager, `cuda_graphs=False`), sync against async. There is +no eager baseline without speculation, so each async percentage is against the +sync value of the same mode: + +| Requests | Prompt | Generated | Pipeline, sync | Pipeline, async | Block, sync | Block, async | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | 1024 | 128 | 5.50 | 5.79 (+5 %) | 6.33 | 6.26 (−1 %) | +| 16 | 1024 | 128 | 9.18 | 8.97 (−2 %) | 10.87 | 10.75 (−1 %) | +| 64 | 937 | 128 | 17.53 | 16.94 (−3 %) | 22.48 | 21.43 (−5 %) | +| 8 | 7500 | 128 | 23.94 | 23.71 (−1 %) | 25.69 | 25.71 (+0.1 %) | + +Reading the numbers: + +- With CUDA Graphs, the asynchronous pipeline is the fastest form in every + shape. Most of its gain is in decode: at 64 requests the time per output token + p50 falls from 129 ms to 64 ms. At 8 × 7500 the run is mostly prefill (17 of + 21 s), which speculation does not change. +- The single-request eager pipeline is the only arm where the asynchronous form + is slower on average (+5 %). The repetitions overlap: synchronous 5.20–5.80 s, + asynchronous 5.50–6.02 s. In an earlier container the same arm was 4 % faster + than synchronous. The cause is not measured. One candidate is the fixed set of + about 50 small operations in a batched pipeline step, whose share is largest + for one request. Before the batch + update, the asynchronous eager pipeline launched about 35 operations for each + request in each step and was 2 %, 10 % and 27 % slower than synchronous at 1, + 16 and 64 requests. +- The synchronous Graph pipeline captures one graph per row count, up to + `cuda_graph_max_graphs` (16). At 64 requests, most of its calls ran eager. +- Eager arms depend on the host: the single-request synchronous eager pipeline + took 5.5 s to 9.5 s on different H100 containers. Compare arms only within one + table. These runs use a different harness and host than the 2D and 3A tables + above, so the absolute numbers do not compare with those tables. +- Not covered: HTTP serving and open-loop arrival traces. All requests are + submitted at once to the engine. + ### Not yet performed - A runtime-to-runtime comparison against vLLM once a Dory-capable build is @@ -402,7 +565,7 @@ Reading the numbers: - Graph-captured prefill, open-loop arrival traces and sampled decoding on the real checkpoint in Graph or async mode. The measurements above are closed-loop, greedy, and their time to first token is eager prefill. -- Self-speculation with CUDA graphs, asynchronous scheduling, preemption, - sampled token-pipeline, adaptive exit, or a draft width above the fixed K. +- Self-speculation with preemption, sampled asynchronous rounds, sampled + token-pipeline, adaptive exit, or a draft width above the fixed K. - Sampled speculation on the real checkpoint (the rejection-sampling contract is qualified on the tiny CPU model only). From 3f3590a2a1420d973e36a0657eef76b78f376003 Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Wed, 18 Nov 2026 01:47:24 -0500 Subject: [PATCH 5/7] docs(dory): report speculation gains against a full graph cache The previous wall times compared speculation with ordinary async Graph decode that used the default graph cache of 16 shapes. At 64 requests that baseline ran eager in most calls, so the reported gain was too large. Replace the results with a concurrency sweep at a fixed prompt length (1, 4, 16 and 64 requests) and a separate long-context run. Every Graph arm now captures up to 128 shapes and has no graph fallback. Add the time to the first token and the time per output token, the GSM8K accuracy, and the known issues: async Graph block at 64 requests and eager async rounds at low concurrency. --- docs/dory.md | 143 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 85 insertions(+), 58 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 73d42da3..8ee6594c 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -479,79 +479,106 @@ so its acceptance differs. Its tokens do not. **GSM8K, BF16.** The first 200 test questions, 8-shot prompt from the first eight training examples, at most 256 generated tokens. An answer is correct -when the number after `####` equals the reference. All 200 requests are -submitted at once, at most 64 in flight: +when the number after `####` equals the reference: -| Arm | Correct | Wall time (s) | Tokens identical to the baseline | -| --- | --- | --- | --- | -| Baseline: async Graph, no speculation | 161/200 | 188 | — | -| Async Graph pipeline | 164/200 | 78 | 32/200 | -| Async Graph block | 163/200 | 77 | 34/200 | +| Arm | Correct | +| --- | --- | +| Baseline: async Graph, no speculation | 161/200 | +| Async Graph pipeline | 164/200 | +| Async Graph block | 163/200 | BF16 tokens depend on the batch composition, so most sequences differ from the baseline after some token. Accuracy does not drop. -**BF16 wall time.** Two loops, a one-loop draft. "Requests" is the number of -concurrent requests, "Prompt" the prompt length of each request in tokens, and -"Generated" the number of tokens that each request generates. All requests are -submitted at once. Each arm runs one warmup and then five timed repetitions in a -row; values are the mean whole-run time in seconds, prefill included. All arms -ran in one container. +**BF16 wall time.** Two loops, a one-loop draft. All requests are sent at the +same time. Each arm runs one warmup, then five timed repetitions, all in one +container. Wall times are the mean whole-run time in seconds, prefill included. +Every Graph arm uses `cuda_graph_max_graphs=128`, so no arm runs eager because +its graph cache is full; every Graph arm had zero graph fallbacks. BF16 +acceptance is 0.85–0.88. -With CUDA Graphs (`cuda_graphs=True` in every column, the baseline included). -Percentages are against the baseline: +Table 1: with CUDA Graphs, concurrency sweep. Prompt 1024 tokens, 128 generated +tokens for each request. Percentages are against the baseline. -| Requests | Prompt | Generated | Baseline: async Graph, no speculation | Sync Graph pipeline | Async Graph pipeline | Async Graph block | -| --- | --- | --- | --- | --- | --- | --- | -| 1 | 1024 | 128 | 2.48 | 1.80 (−28 %) | **1.77 (−29 %)** | 1.91 (−23 %) | -| 16 | 1024 | 128 | 5.84 | 5.61 (−4 %) | **5.27 (−10 %)** | 5.61 (−4 %) | -| 64 | 937 | 128 | 22.19 | 16.72 (−25 %) | **13.61 (−39 %)** | 14.96 (−33 %) | -| 8 | 7500 | 128 | 20.79 | 20.20 (−3 %) | **20.15 (−3 %)** | 20.36 (−2 %) | +| Requests | Baseline (async Graph, no spec) | Baseline, default graph cache | Sync Graph pipeline | Async Graph pipeline | Async Graph block | +| --- | --- | --- | --- | --- | --- | +| 1 | 2.50 | 2.49 (−0.1 %) | 1.83 (−27 %) | **1.78 (−29 %)** | 1.91 (−23 %) | +| 4 | 3.17 | 3.18 (+0.4 %) | 2.66 (−16 %) | **2.63 (−17 %)** | 2.95 (−7 %) | +| 16 | 5.89 | 5.90 (+0.1 %) | 5.51 (−6 %) | **5.30 (−10 %)** | 5.63 (−5 %) | +| 64 | 15.60 | 25.72 (+65 %) | 16.00 (+3 %) | **15.10 (−3 %)** | 16.45 (+5 %) | -Time to first token p50 / time per output token p50 / p90, in seconds and -milliseconds: +Table 2: latency with CUDA Graphs, same workload. TTFT is the time to the first +token; TPOT is the time per output token after the first token. Percentages are +against the baseline at the same concurrency. -| Requests | Prompt | Generated | Baseline | Async Graph pipeline | Async Graph block | +| Requests | Arm | TTFT p50 (s) | TTFT p90 (s) | TPOT p50 (ms) | TPOT p90 (ms) | | --- | --- | --- | --- | --- | --- | -| 1 | 1024 | 128 | 0.17 / 18.2 / 18.2 | 0.17 / 12.6 / 12.6 | 0.17 / 13.7 / 13.7 | -| 16 | 1024 | 128 | 1.97 / 35.0 / 40.6 | 1.65 / 28.6 / 35.9 | 1.79 / 30.0 / 36.1 | -| 64 | 937 | 128 | 5.86 / 129.3 / 155.7 | 5.35 / 63.7 / 92.6 | 6.16 / 61.8 / 87.9 | -| 8 | 7500 | 128 | 17.56 / 32.9 / 47.6 | 17.19 / 28.3 / 49.6 | 17.21 / 29.0 / 48.7 | +| 1 | Baseline (async Graph, no spec) | 0.18 | 0.18 | 18.3 | 18.3 | +| 1 | Async Graph pipeline | 0.17 (−2 %) | 0.20 (+13 %) | 12.6 (−31 %) | 12.6 (−31 %) | +| 1 | Async Graph block | 0.17 (−1 %) | 0.19 (+7 %) | 13.7 (−25 %) | 13.7 (−25 %) | +| 4 | Baseline (async Graph, no spec) | 0.67 | 0.67 | 20.0 | 20.0 | +| 4 | Async Graph pipeline | 0.66 (−1 %) | 0.67 (−0.8 %) | 16.6 (−17 %) | 17.4 (−13 %) | +| 4 | Async Graph block | 0.69 (+3 %) | 0.69 (+3 %) | 18.1 (−10 %) | 19.0 (−5 %) | +| 16 | Baseline (async Graph, no spec) | 2.01 | 2.75 | 35.2 | 40.9 | +| 16 | Async Graph pipeline | 1.68 (−17 %) | 2.70 (−2 %) | 28.8 (−18 %) | 36.1 (−12 %) | +| 16 | Async Graph block | 1.79 (−11 %) | 2.93 (+6 %) | 30.1 (−15 %) | 36.1 (−12 %) | +| 64 | Baseline (async Graph, no spec) | 6.21 | 10.58 | 76.6 | 108.5 | +| 64 | Async Graph pipeline | 5.78 (−7 %) | 10.02 (−5 %) | 70.9 (−7 %) | 102.5 (−5 %) | +| 64 | Async Graph block | 6.43 (+3 %) | 11.54 (+9 %) | 68.1 (−11 %) | 96.4 (−11 %) | + +Table 3: without CUDA Graphs (eager), sync against async, same workload. There +is no eager baseline without speculation, so each async percentage is against +the sync value of the same mode. + +| Requests | Pipeline sync | Pipeline async | Block sync | Block async | +| --- | --- | --- | --- | --- | +| 1 | 6.80 | 6.42 (−6 %) | 6.83 | 7.16 (+5 %) | +| 4 | 7.80 | 8.24 (+6 %) | 9.64 | 9.71 (+0.8 %) | +| 16 | 10.74 | 10.57 (−2 %) | 13.24 | 13.25 (+0.1 %) | +| 64 | 21.22 | 20.49 (−3 %) | 28.02 | 27.29 (−3 %) | -Without CUDA Graphs (eager, `cuda_graphs=False`), sync against async. There is -no eager baseline without speculation, so each async percentage is against the -sync value of the same mode: +Table 4: long context. 8 requests, prompt 8192 tokens, 128 generated tokens for +each request. -| Requests | Prompt | Generated | Pipeline, sync | Pipeline, async | Block, sync | Block, async | -| --- | --- | --- | --- | --- | --- | --- | -| 1 | 1024 | 128 | 5.50 | 5.79 (+5 %) | 6.33 | 6.26 (−1 %) | -| 16 | 1024 | 128 | 9.18 | 8.97 (−2 %) | 10.87 | 10.75 (−1 %) | -| 64 | 937 | 128 | 17.53 | 16.94 (−3 %) | 22.48 | 21.43 (−5 %) | -| 8 | 7500 | 128 | 23.94 | 23.71 (−1 %) | 25.69 | 25.71 (+0.1 %) | +| Arm | Wall (s) | TTFT p50 (s) | TTFT p90 (s) | TPOT p50 (ms) | TPOT p90 (ms) | +| --- | --- | --- | --- | --- | --- | +| Baseline (async Graph, no spec) | 23.73 | 18.60 | 20.71 | 53.2 | 68.9 | +| Sync Graph pipeline | 23.52 (−0.9 %) | 17.85 (−4 %) | 20.73 (+0.1 %) | 51.2 (−4 %) | 72.5 (+5 %) | +| Async Graph pipeline | 23.18 (−2 %) | 17.69 (−5 %) | 20.58 (−0.6 %) | 49.4 (−7 %) | 70.3 (+2 %) | +| Async Graph block | 23.13 (−3 %) | 17.60 (−5 %) | 20.42 (−1 %) | 49.9 (−6 %) | 67.9 (−1 %) | +| Sync eager pipeline | 28.83 | 18.04 | 21.08 | 86.1 | 111.4 | +| Async eager pipeline | 28.52 (−1 % vs sync) | 18.10 (+0.3 % vs sync) | 21.15 (+0.3 % vs sync) | 87.2 (+1 % vs sync) | 108.0 (−3 % vs sync) | Reading the numbers: -- With CUDA Graphs, the asynchronous pipeline is the fastest form in every - shape. Most of its gain is in decode: at 64 requests the time per output token - p50 falls from 129 ms to 64 ms. At 8 × 7500 the run is mostly prefill (17 of - 21 s), which speculation does not change. -- The single-request eager pipeline is the only arm where the asynchronous form - is slower on average (+5 %). The repetitions overlap: synchronous 5.20–5.80 s, - asynchronous 5.50–6.02 s. In an earlier container the same arm was 4 % faster - than synchronous. The cause is not measured. One candidate is the fixed set of - about 50 small operations in a batched pipeline step, whose share is largest - for one request. Before the batch - update, the asynchronous eager pipeline launched about 35 operations for each - request in each step and was 2 %, 10 % and 27 % slower than synchronous at 1, - 16 and 64 requests. -- The synchronous Graph pipeline captures one graph per row count, up to - `cuda_graph_max_graphs` (16). At 64 requests, most of its calls ran eager. -- Eager arms depend on the host: the single-request synchronous eager pipeline - took 5.5 s to 9.5 s on different H100 containers. Compare arms only within one - table. These runs use a different harness and host than the 2D and 3A tables - above, so the absolute numbers do not compare with those tables. -- Not covered: HTTP serving and open-loop arrival traces. All requests are - submitted at once to the engine. +- The gain of speculation falls as concurrency rises. At one request the GPU + has spare capacity, so draft and verify work is almost free. At 64 requests + the GPU is busy, and the extra verify rows cancel most of the saved steps. +- At 8 × 8192, prefill takes about 18 of 24 seconds. Speculation shortens only + decode, so the whole run is 2–3 % faster. +- The baseline with the default graph cache (16 shapes) runs eager in most calls + at 64 requests and takes 25.72 s instead of 15.60 s. Comparisons against that + baseline overstate the gain of speculation. +- These runs use a different harness and host than the 2D and 3A tables above, + so the absolute numbers do not compare with those tables. +- Not covered: HTTP serving and open-loop arrival traces. + +**Known issues.** + +- Async Graph block is 5 % slower than the baseline at 64 requests. Its time per + output token is 11 % lower, but its time to the first token is 3–9 % higher. + The async Graph pipeline is 3 % faster there, so it is the better mode at high + concurrency. +- Without CUDA Graphs, async rounds at 1 and 4 requests are between 6 % faster + and 6 % slower than sync rounds (Table 3). A dedicated run alternated a sync + and an async engine in ABBA order, 10 timed repetitions each, at one request + in pipeline mode. Async was 2.7 % and 3.9 % slower in two containers. The + profiler counts 5503 kernel launches per generated token for sync and 5626 for + async (+124, +2.2 %). Async rounds keep positions on the GPU, so the GPU + computes the KV metadata, the PRELUDE and CODA write masks and the slot-pool + update; sync rounds do this work on the CPU. Eager Dory is bound by host-side + launch cost, so the extra launches give the extra time. At 16 and 64 requests, + async eager is 0–3 % faster than sync eager. ### Not yet performed From d74adfac7376a572f3620c331cf145592a3dea04 Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Sat, 21 Nov 2026 15:48:22 -0500 Subject: [PATCH 6/7] perf(dory): share one boundary view in asynchronous pipeline steps PRELUDE and CODA of a pipeline step built two depth-0 views with the same positions and tables. Only the write masks were different. CODA now reuses the PRELUDE view with its own write mask, as synchronous steps and asynchronous block rounds already do. This also removes 11 kernel launches per generated token in eager async rounds at one request. Fresh rows reach the draft depth, so the CODA mask is the PRELUDE mask plus the active rows that reach the target depth. --- vllm_rlt/worker/async_speculative.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vllm_rlt/worker/async_speculative.py b/vllm_rlt/worker/async_speculative.py index 3a06e42a..fac805a5 100644 --- a/vllm_rlt/worker/async_speculative.py +++ b/vllm_rlt/worker/async_speculative.py @@ -934,11 +934,11 @@ def _pipeline_step(self, spans, states, tables, allocations, buffer): ) # Only rows that reach the draft or target depth read a distribution. # The output-stage KV of other rows must stay the KV of their draft pass. - reached = row_depths + self.config.draft_loops - endpoints = (reached == self.config.draft_loops) | (reached == self.config.target_loops) - coda_kv = self.device_metadata( - positions, row_tables, 0, allocations, row_active, row_active & endpoints - ) + # Fresh rows reach the draft depth, so CODA writes the PRELUDE rows and + # the rows that reach the target depth. Both stages reuse one depth-0 + # view because eager rounds are bound by kernel launches. + last = row_depths == self.config.target_loops - self.config.draft_loops + coda_kv = replace(prelude_kv, write_mask=prelude_kv.write_mask | (row_active & last)) prelude = self._prelude(tokens.view(-1)[flat], prelude_kv) hidden = torch.where(fresh[:, None], prelude, retained.view(n * width, -1)[flat]) for offset in range(self.config.draft_loops): From 4435bdd85443046e981d45b942574446596dc263 Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Sat, 21 Nov 2026 15:48:25 -0500 Subject: [PATCH 7/7] docs(dory): describe the shared boundary view and the discarded stage work Say that PRELUDE and CODA of a pipeline step use one depth-0 view, as the other speculative paths do, and which boundary outputs the runtime discards. This extra work can explain a part of the small gain at 64 requests. Add the new launch count and ABBA timing for eager async rounds at one request. --- docs/dory.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 8ee6594c..b0fda856 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -245,6 +245,10 @@ next round is the serial target history. the rows that store KV. PRELUDE stores only for rows at depth 0. CODA stores only for rows that reach the draft or the target depth. A row between these depths (1→4) keeps the CODA KV of its draft pass, as in a synchronous step. + Both stages use one depth-0 view and differ only in the write mask, as in + synchronous steps and asynchronous block rounds. + The runtime discards the PRELUDE output of rows that are not at depth 0, and + the CODA output of rows that reach neither depth. - **Sampling.** Asynchronous rounds take greedy requests only. Asynchronous block rounds can sample for models without boundary KV, but that combination is not qualified for Dory. Such a request fails when it is added. @@ -495,7 +499,8 @@ same time. Each arm runs one warmup, then five timed repetitions, all in one container. Wall times are the mean whole-run time in seconds, prefill included. Every Graph arm uses `cuda_graph_max_graphs=128`, so no arm runs eager because its graph cache is full; every Graph arm had zero graph fallbacks. BF16 -acceptance is 0.85–0.88. +acceptance is 0.85–0.88. Tables 1–4 were measured before PRELUDE and CODA of a +pipeline step shared one depth-0 view. Table 1: with CUDA Graphs, concurrency sweep. Prompt 1024 tokens, 128 generated tokens for each request. Percentages are against the baseline. @@ -554,6 +559,11 @@ Reading the numbers: - The gain of speculation falls as concurrency rises. At one request the GPU has spare capacity, so draft and verify work is almost free. At 64 requests the GPU is busy, and the extra verify rows cancel most of the saved steps. +- In pipeline mode, every row runs both boundary stages, so that each call keeps + one shape. PRELUDE also runs on rows that already hold their input-stage KV, + and CODA also runs on rows that reach neither the draft nor the target depth. + The runtime discards these outputs. At 64 requests, this extra work can + explain a part of the small gain. - At 8 × 8192, prefill takes about 18 of 24 seconds. Speculation shortens only decode, so the whole run is 2–3 % faster. - The baseline with the default graph cache (16 shapes) runs eager in most calls @@ -573,12 +583,19 @@ Reading the numbers: and 6 % slower than sync rounds (Table 3). A dedicated run alternated a sync and an async engine in ABBA order, 10 timed repetitions each, at one request in pipeline mode. Async was 2.7 % and 3.9 % slower in two containers. The - profiler counts 5503 kernel launches per generated token for sync and 5626 for - async (+124, +2.2 %). Async rounds keep positions on the GPU, so the GPU + profiler counted 5503 kernel launches per generated token for sync and 5626 + for async (+124, +2.2 %). Async rounds keep positions on the GPU, so the GPU computes the KV metadata, the PRELUDE and CODA write masks and the slot-pool update; sync rounds do this work on the CPU. Eager Dory is bound by host-side launch cost, so the extra launches give the extra time. At 16 and 64 requests, async eager is 0–3 % faster than sync eager. +- After PRELUDE and CODA of a pipeline step shared one depth-0 view, async makes + 5615 launches per generated token (+113 against sync). The same ABBA run gave + −0.2 % and +0.4 % in two new containers, and async was slower in 5 of 10 + rounds in each. The time between containers varies by a few percent, so these + runs do not show that the gap is closed. A fused kernel for the per-step + update and metadata would remove more launches. It also changes the Ouro path, + so it is a separate change. ### Not yet performed