Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ and coda, including the LM head.
The [Dory baseline notes](dory.md) list the narrower scope qualified for Dory:
synchronous fixed-loop execution with boundary KV in both KV modes, sliding-window
masking on the Torch and Triton backends, and BF16 agreement with the pinned
checkpoint at 1/2/4 loops; Graphs, WCPB, async, speculation, prefix reuse and PD
remain out of scope.
checkpoint at 1/2/4 loops. The notes also list the qualified scope of Graphs,
async, speculation, prefix reuse and PD; WCPB remains out of scope.
CPU execution provides a Torch reference backend. FlashAttention hardware validation is currently documented
for FA4 on B300; FA2/FA3 require validation on their target devices. Disaggregated
serving currently targets multiple GPUs on a single host.
Expand Down
254 changes: 227 additions & 27 deletions docs/dory.md

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions tests/test_backend_selection.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,8 @@ def plan_cache(
speculative=None,
staged_prefill=None,
fp8_scale=None,
layer_windows=None,
reclaim_swa=False,
):
assert selected == "triton" # FA2 cannot use the configured 16-token pages.
assert staged_prefill is False
Expand Down
14 changes: 12 additions & 2 deletions tests/test_cdb_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -686,8 +686,18 @@ def test_cuda_graph_reserve_reduces_budget_with_fixed_memory_inputs(monkeypatch,
scheduler = SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=3)
execution = ExecutionConfig(cuda_graph_memory_reserve_bytes=32 * mib)

_, eager = plan_cache(base, cache, scheduler, execution, "triton")
_, graphed = plan_cache(base, cache, scheduler, replace(execution, cuda_graphs=True), "triton")
_, eager = plan_cache(
base, cache, scheduler, execution, "triton", layer_windows=None, reclaim_swa=False
)
_, graphed = plan_cache(
base,
cache,
scheduler,
replace(execution, cuda_graphs=True),
"triton",
layer_windows=None,
reclaim_swa=False,
)

assert eager["cuda_graph_reserve_bytes"] == 0
assert graphed["cuda_graph_reserve_bytes"] == 32 * mib
Expand Down
18 changes: 16 additions & 2 deletions tests/test_device.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,23 @@ def test_npu_requires_bf16_kv_before_allocation(backend):
def test_npu_does_not_use_cpu_default_memory_budget():
model = npu_model_stub()
with pytest.raises(ValueError, match="explicit num_blocks"):
plan_cache(model, CacheConfig(), SchedulerConfig(), ExecutionConfig(), "torch")
plan_cache(
model,
CacheConfig(),
SchedulerConfig(),
ExecutionConfig(),
"torch",
layer_windows=None,
reclaim_swa=False,
)
blocks, info = plan_cache(
model, CacheConfig(num_blocks=17), SchedulerConfig(), ExecutionConfig(), "torch"
model,
CacheConfig(num_blocks=17),
SchedulerConfig(),
ExecutionConfig(),
"torch",
layer_windows=None,
reclaim_swa=False,
)
assert blocks == 17 and info["source"] == "blocks"

Expand Down
244 changes: 238 additions & 6 deletions tests/test_dory.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@
import pytest
import torch

from tests.helpers import tiny_ouro_config
from tests.reference.dory import dense_dory_reference
from vllm_rlt import CacheConfig, ExecutionConfig, SamplingParams, SchedulerConfig
from vllm_rlt.attention import BackendCapabilities
from vllm_rlt.attention.backend import AttentionBackend
from vllm_rlt.core.kv_cache_manager import KVCacheManager
from vllm_rlt.engine.llm_engine import LLMEngine
from vllm_rlt.kernels.paged_attention import torch_paged_attention
from vllm_rlt.models import AutoModelForCausalLM, DoryConfig, DoryForCausalLM
from vllm_rlt.models import AutoModelForCausalLM, DoryConfig, DoryForCausalLM, OuroForCausalLM

ARTIFACT_PAIRS = 36

Expand Down Expand Up @@ -111,9 +112,11 @@ def model():
return DoryForCausalLM(tiny_dory_config()).eval()


def make_cache(model, layout=None, block_size=2):
def make_cache(model, layout=None, block_size=2, enable_prefix_caching=False, grouped=False):
config = model.config
return KVCacheManager(
layer_windows=model.kv_layer_windows if grouped else None,
enable_prefix_caching=enable_prefix_caching,
num_layers=config.num_kv_layers,
num_kv_heads=config.num_key_value_heads,
head_dim=config.head_dim,
Expand All @@ -128,10 +131,10 @@ def make_cache(model, layout=None, block_size=2):
)


def run_one_token_per_step(model, tokens):
def run_one_token_per_step(model, tokens, cache=None):
"""Execute the declared reference schedule: one token row at a time."""
config = model.config
cache = make_cache(model)
cache = make_cache(model) if cache is None else cache
assert cache.allocate("request", len(tokens))
loops = [[] for _ in range(config.total_ut_steps)]
logits = []
Expand All @@ -148,10 +151,10 @@ def run_one_token_per_step(model, tokens):
return {"logits": torch.stack(logits), "loops": [torch.stack(item) for item in loops]}, cache


def run_all_positions_at_once(model, tokens):
def run_all_positions_at_once(model, tokens, cache=None):
"""Same schedule, but every stage call carries all causal positions."""
config = model.config
cache = make_cache(model)
cache = make_cache(model) if cache is None else cache
assert cache.allocate("request", len(tokens))
positions = list(range(len(tokens)))
ids = ["request"] * len(tokens)
Expand Down Expand Up @@ -690,3 +693,232 @@ def test_dory_packed_prefill_rejects_graphs(model):
model,
execution_config=ExecutionConfig(packed_prefill=True, cuda_graphs=True),
)


def prefill_rows(model, cache, request_id, tokens, start):
"""Run positions ``start:`` one row at a time on an allocated request."""
config = model.config
logits = []
for position in range(start, len(tokens)):
boundary = cache._prepare_batch([request_id], [0], [position])
hidden = model.prelude_prepared(tokens[position : position + 1], boundary, cache)
for depth in range(config.total_ut_steps):
core = cache._prepare_batch([request_id], [depth], [position])
hidden, _ = model.recurrent_prepared(hidden, core, cache)
logits.append(model.coda_prepared(hidden, boundary, cache)[0])
cache.finalize_token(request_id, position, config.total_ut_steps - 1)
return torch.stack(logits)


@pytest.mark.parametrize("kv_mode", ["per_loop", "last_loop"])
@pytest.mark.parametrize("grouped", [False, True], ids=["legacy", "grouped"])
@pytest.mark.parametrize("loops", [1, 2, 4])
def test_prefix_hit_resumes_boundary_and_recurrent_kv(loops, grouped, kv_mode):
torch.manual_seed(42)
model = DoryForCausalLM(
tiny_dory_config(n_recurrent_loops=loops, recurrent_kv_cache_mode=kv_mode)
).eval()
cache = make_cache(model, enable_prefix_caching=True, grouped=grouped)
# The cached prefix exceeds the four-token sliding attention window.
tokens = torch.tensor([4, 7, 3, 9, 5, 2, 8, 6, 3])
expected = dense_dory_reference(model, tokens)["logits"]

assert cache.allocate("cold", len(tokens))
torch.testing.assert_close(
prefill_rows(model, cache, "cold", tokens, 0), expected, atol=3e-6, rtol=3e-5
)
cache.publish_prefix("cold", tokens.tolist(), len(tokens))
# The last prompt token is always recomputed, so four whole pages publish.
assert len(cache._prefixes) == 4
cache.free("cold")

prefix = cache.lookup_prefix(tokens.tolist())
assert len(prefix) == 4
assert cache.allocate("hit", len(tokens), prefix=prefix)
written = cache._get_allocation("hit").written
for plane, layers in enumerate(written):
for layer, positions in enumerate(layers):
owns = plane == 0 or layer in model.recurrent_kv_layers
# Boundary KV is restored only where the cold run wrote it.
assert positions.prefix == (8 if owns else 0)
torch.testing.assert_close(
prefill_rows(model, cache, "hit", tokens, 8), expected[8:], atol=3e-6, rtol=3e-5
)
cache.free("hit")
assert cache.num_used_blocks == 0


@pytest.mark.parametrize("grouped", [False, True], ids=["legacy", "grouped"])
@pytest.mark.parametrize("missing", ["boundary_plane0", "recurrent_plane0", "recurrent_deep_plane"])
def test_prefix_publication_requires_every_owned_plane(model, missing, grouped):
cache = make_cache(model, enable_prefix_caching=True, grouped=grouped)
tokens = torch.tensor([4, 7, 3, 9, 5])
assert cache.allocate("request", len(tokens))
prefill_rows(model, cache, "request", tokens, 0)
layers = cache._get_allocation("request").written
boundary = model.prelude_kv_layers[0]
recurrent = model.recurrent_kv_layers[0]
plane, layer = {
"boundary_plane0": (0, boundary),
"recurrent_plane0": (0, recurrent),
"recurrent_deep_plane": (cache.storage_depths - 1, recurrent),
}[missing]
# Mark page 1 (positions 2-3) incomplete in one plane: only page 0 may publish.
layers[plane][layer].prefix = 2
cache.publish_prefix("request", tokens.tolist(), len(tokens))
assert len(cache._prefixes) == 1


def test_grouped_prefix_pages_outlive_the_publishing_request(model):
cache = make_cache(model, enable_prefix_caching=True, grouped=True)
tokens = torch.tensor([4, 7, 3, 9, 5])
expected = dense_dory_reference(model, tokens)["logits"]
assert cache.allocate("cold", len(tokens))
prefill_rows(model, cache, "cold", tokens, 0)
cache.publish_prefix("cold", tokens.tolist(), len(tokens))
published = {b for blocks in cache._prefixes.values() for b in blocks}
# Two token pages, each one physical page per (group, plane, layer).
assert len(published) == 2 * cache.pages_per_token_block
cache.free("cold")
# Allocate another request; the published pages must not be handed out.
assert cache.allocate("filler", len(tokens))
assert not published & set(
b
for layer in cache.layer_groups
for depth in range(cache.max_loops if cache.layer_group(layer)[1].recurrent else 1)
for b in cache.layer_block_table("filler", layer, depth)
)
cache.free("filler")
prefix = cache.lookup_prefix(tokens.tolist())
assert cache.allocate("hit", len(tokens), prefix=prefix)
for layer in cache.layer_groups:
assert cache.retained_range("hit", layer, 0) == (0, 4)
torch.testing.assert_close(
prefill_rows(model, cache, "hit", tokens, 4), expected[4:], atol=3e-6, rtol=3e-5
)
cache.free("hit")
assert cache.num_used_blocks == 0


def run_engine(engine, requests, params):
for request_id, prompt in requests:
engine.add_request(request_id, prompt, params)
finished = {}
while engine.has_unfinished_requests():
for output in engine.step():
if output.finished:
finished[output.request_id] = output.token_ids
return finished


def greedy_reference(model, prompt, steps):
tokens = list(prompt)
for _ in range(steps):
logits = dense_dory_reference(model, torch.tensor(tokens))["logits"]
tokens.append(int(logits[-1].argmax()))
return tokens[len(prompt) :]


@pytest.mark.parametrize("kv_mode,chunk", [("per_loop", 4), ("last_loop", 3), ("last_loop", 4)])
@pytest.mark.parametrize("loops", [1, 2, 4])
def test_engine_prefix_hits_match_cold_greedy_reference(loops, kv_mode, chunk):
torch.manual_seed(42)
model = DoryForCausalLM(
tiny_dory_config(n_recurrent_loops=loops, recurrent_kv_cache_mode=kv_mode)
).eval()
engine = LLMEngine(
model,
cache_config=CacheConfig(
num_blocks=512,
block_size=2,
enable_prefix_caching=True,
layout=model.kv_cache_layout,
),
scheduler_config=SchedulerConfig(
max_num_seqs=2, max_num_batched_tokens=4, prefill_chunk_size=chunk
),
)
steps = model.config.total_ut_steps
params = SamplingParams(max_tokens=3, min_loops=steps, max_loops=steps, ignore_eos=True)
shared = [4, 7, 3, 9, 5, 2, 8, 6]
prompts = {
"cold": shared + [3],
# Block-aligned: the last full page is still recomputed.
"aligned": shared + [3, 5],
"branch": shared + [10, 11, 12],
"same": shared + [3],
}
cache = engine.cache_manager
assert run_engine(engine, [("cold", prompts["cold"])], params) == {
"cold": greedy_reference(model, prompts["cold"], 3)
}
assert cache.prefix_hits == 0
# Run the remaining requests concurrently over the shared published prefix.
finished = run_engine(
engine, [(rid, prompts[rid]) for rid in ("aligned", "branch", "same")], params
)
for rid in ("aligned", "branch", "same"):
assert finished[rid] == greedy_reference(model, prompts[rid], 3), rid
assert cache.prefix_hits >= 3 * len(shared)
assert cache.num_used_blocks == 0


@pytest.mark.parametrize(
"execution", [ExecutionConfig(async_scheduling=True), ExecutionConfig(cuda_graphs=True)]
)
def test_last_loop_prefix_caching_requires_sync_eager(execution):
model = DoryForCausalLM(tiny_dory_config(recurrent_kv_cache_mode="last_loop")).eval()
with pytest.raises(ValueError, match="synchronous eager"):
LLMEngine(
model,
execution_config=execution,
cache_config=CacheConfig(
num_blocks=64, block_size=2, enable_prefix_caching=True, layout="shared"
),
)


def test_shared_prefix_caching_requires_an_opted_in_model():
with pytest.raises(ValueError, match="shared-KV prefix caching"):
LLMEngine(
OuroForCausalLM(tiny_ouro_config()),
cache_config=CacheConfig(enable_prefix_caching=True, layout="shared"),
)


@pytest.mark.parametrize("loops", [1, 2, 4])
def test_evicted_prefix_falls_back_to_cold_prefill(loops):
torch.manual_seed(42)
model = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops)).eval()
steps = model.config.total_ut_steps
# Grouped KV stores one layer per physical page: boundary layers own one
# plane, recurrent layers own ``steps``. A configured block is
# ``num_kv_layers`` physical pages. Each request reserves eleven tokens,
# spanning six token pages; those plus four cached pages exceed the pool's eight.
recurrent = set(model.recurrent_kv_layers)
pages_per_token_page = sum(
steps if layer in recurrent else 1 for layer in model.kv_layer_windows
)
engine = LLMEngine(
model,
cache_config=CacheConfig(
num_blocks=8 * pages_per_token_page // model.config.num_kv_layers,
block_size=2,
enable_prefix_caching=True,
),
scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4),
)
cache = engine.cache_manager
params = SamplingParams(max_tokens=3, min_loops=steps, max_loops=steps, ignore_eos=True)
prompt = [4, 7, 3, 9, 5, 2, 8, 6, 3]
other = [10, 11, 12, 13, 14, 15, 16, 17, 18]
expected = greedy_reference(model, prompt, 3)

assert run_engine(engine, [("cold", prompt)], params) == {"cold": expected}
assert len(cache.lookup_prefix(prompt)) == 4
run_engine(engine, [("other", other)], params)
assert cache.lookup_prefix(prompt) == ()
hits = cache.prefix_hits
assert run_engine(engine, [("again", prompt)], params) == {"again": expected}
assert cache.prefix_hits == hits
assert cache.num_used_blocks == 0
Loading