From bd4a577d6270bc181abca0a2dcd9d8471f7d58ee Mon Sep 17 00:00:00 2001 From: bohaotu Date: Fri, 20 Nov 2026 14:27:50 +0800 Subject: [PATCH 1/6] feat(medusa): add candidate-tree consumer and acceptance evaluator Port the reference tree buffers, candidate construction and greedy acceptance, and add a standalone Hugging Face Target evaluator that reports acceptance length, greedy decode equality and repeat stability. Tree buffers match the pinned Medusa revision exactly; the evaluator is a reference component, not a serving runtime. --- aspec/commands/evaluate_medusa_acceptance.py | 136 +++++++ aspec/inference/medusa_consumer.py | 401 +++++++++++++++++++ docs/support.md | 2 +- docs/user-guide/algorithms.md | 5 +- docs/user-guide/medusa.md | 20 +- tests/cpu/test_medusa_consumer.py | 119 ++++++ tools/evaluate_medusa_acceptance.py | 13 + 7 files changed, 691 insertions(+), 5 deletions(-) create mode 100644 aspec/commands/evaluate_medusa_acceptance.py create mode 100644 aspec/inference/medusa_consumer.py create mode 100644 tests/cpu/test_medusa_consumer.py create mode 100644 tools/evaluate_medusa_acceptance.py diff --git a/aspec/commands/evaluate_medusa_acceptance.py b/aspec/commands/evaluate_medusa_acceptance.py new file mode 100644 index 0000000..dee094c --- /dev/null +++ b/aspec/commands/evaluate_medusa_acceptance.py @@ -0,0 +1,136 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Measure Medusa greedy acceptance and decode equality with a Hugging Face Target.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import torch + + +def _choices(path, heads): + if path is not None: + choices = json.loads(Path(path).read_text()) + else: + from aspec.inference.medusa_consumer import DEFAULT_MEDUSA_CHOICES + + choices = DEFAULT_MEDUSA_CHOICES + filtered = [list(item) for item in choices if len(item) <= heads] + if not filtered: + raise ValueError("No medusa_choices path fits the draft's head depth") + return filtered + + +def _parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--draft", type=Path, required=True) + parser.add_argument("--target", required=True) + parser.add_argument("--prompts", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--limit", type=int, default=8) + parser.add_argument("--max-new-tokens", type=int, default=32) + parser.add_argument("--choices", type=Path, default=None) + parser.add_argument("--top-k", type=int, default=10) + parser.add_argument("--verify-batch-size", type=int, default=16) + parser.add_argument("--device", default=None) + parser.add_argument( + "--dtype", choices=("bfloat16", "float16", "float32"), default="bfloat16") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--enable-thinking", action="store_true") + return parser.parse_args() + + +def main(): + args = _parse_args() + from transformers import AutoModelForCausalLM, AutoTokenizer + + from aspec.commands.evaluate_draft import load_prompts + from aspec.draft_artifact import load_native_draft + from aspec.inference.medusa_consumer import HFCausalVerifier, MedusaConsumer + + device = args.device or ("cuda" if torch.cuda.is_available() else "cpu") + dtype = getattr(torch, args.dtype) + tokenizer = AutoTokenizer.from_pretrained( + args.target, trust_remote_code=args.trust_remote_code) + target = AutoModelForCausalLM.from_pretrained( + args.target, torch_dtype=dtype, trust_remote_code=args.trust_remote_code) + target.to(device).eval() + draft = load_native_draft(args.draft, device=device, dtype=dtype) + choices = _choices(args.choices, draft.medusa_num_heads) + verifier = HFCausalVerifier( + target, device=device, verify_batch_size=args.verify_batch_size) + consumer = MedusaConsumer( + draft, verifier, medusa_choices=choices, top_k=args.top_k) + prompts = load_prompts( + args.prompts, tokenizer, args.limit, + enable_thinking=True if args.enable_thinking else None) + + eos = tokenizer.eos_token_id + results = [] + for ids in prompts: + prefix = torch.tensor(ids, dtype=torch.long) + baseline = verifier.greedy_generate( + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos) + speculative = consumer.generate( + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos) + repeat = consumer.generate( + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos) + start = len(ids) + results.append({ + "prompt_sha256": hashlib.sha256( + json.dumps(ids).encode("utf-8")).hexdigest(), + "prompt_tokens": len(ids), + "generated_tokens": speculative["generated_tokens"], + "verifier_calls": speculative["verifier_calls"], + "acceptance_length": speculative["acceptance_length"], + "decode_equal": ( + speculative["token_ids"][start:].tolist() + == baseline["token_ids"][start:].tolist()), + "repeat_stable": ( + speculative["token_ids"].tolist() == repeat["token_ids"].tolist()), + }) + + total_tokens = sum(item["generated_tokens"] for item in results) + total_calls = sum(item["verifier_calls"] for item in results) + if total_calls < 1: + raise ValueError("Acceptance measurement produced no verifier calls") + choices_blob = json.dumps(choices, sort_keys=True, separators=(",", ":")) + report = { + "schema_version": "aspec.medusa-acceptance/v1", + "draft_manifest_sha256": hashlib.sha256( + (args.draft / "export_manifest.json").read_bytes()).hexdigest(), + "target": args.target, + "prompts": str(args.prompts), + "prompt_count": len(results), + "max_new_tokens": args.max_new_tokens, + "top_k": args.top_k, + "choices_sha256": hashlib.sha256(choices_blob.encode("utf-8")).hexdigest(), + "choices_count": len(choices), + "device": device, + "device_name": ( + torch.cuda.get_device_name(torch.device(device)) + if device.startswith("cuda") else None), + "dtype": args.dtype, + "acceptance_length": total_tokens / total_calls, + "acceptance_convention": ( + "generated_tokens / verifier_calls; includes the bonus token"), + "decode_equal": all(item["decode_equal"] for item in results), + "repeat_stable": all(item["repeat_stable"] for item in results), + "results": results, + } + try: + serialized = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + except ValueError as exc: + raise ValueError("Acceptance report contains non-finite metrics") from exc + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(serialized + "\n") + print(serialized) + + +if __name__ == "__main__": + main() diff --git a/aspec/inference/medusa_consumer.py b/aspec/inference/medusa_consumer.py new file mode 100644 index 0000000..97a9dc3 --- /dev/null +++ b/aspec/inference/medusa_consumer.py @@ -0,0 +1,401 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +# Adapted from FasterDecoding/Medusa e2a5d20c048a9b0a4092e6933c34313687422518 +# (medusa/model/utils.py, medusa/model/medusa_choices.py, Apache-2.0). +# Tree buffers, candidate construction and greedy acceptance follow the +# reference; batched path verification and explicit evidence accounting are +# ASpec additions. +"""Medusa candidate-tree proposals, verification and acceptance accounting. + +The consumer is deliberately not a serving runtime: callers provide a verifier +that owns the Target forward pass, KV/cache policy and sampling. The reference +verifier here batches independent teacher-forced paths, which is exact for +greedy verification but makes no throughput claim. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol, Sequence + +import torch +from torch import Tensor + +DEFAULT_TOPK = 10 + +# Reference Medusa tree for Vicuna/7B-style heads (medusa/model/medusa_choices.py). +DEFAULT_MEDUSA_CHOICES = [ + [0], [0, 0], [1], [0, 1], [2], [0, 0, 0], [1, 0], [0, 2], [3], [0, 3], + [4], [0, 4], [2, 0], [0, 5], [0, 0, 1], [5], [0, 6], [6], [0, 7], + [0, 1, 0], [1, 1], [7], [0, 8], [0, 0, 2], [3, 0], [0, 9], [8], [9], + [1, 0, 0], [0, 2, 0], [1, 2], [0, 0, 3], [4, 0], [2, 1], [0, 0, 4], + [0, 0, 5], [0, 0, 0, 0], [0, 1, 1], [0, 0, 6], [0, 3, 0], [5, 0], + [1, 3], [0, 0, 7], [0, 0, 8], [0, 0, 9], [6, 0], [0, 4, 0], [1, 4], + [7, 0], [0, 1, 2], [2, 0, 0], [3, 1], [2, 2], [8, 0], [0, 5, 0], + [1, 5], [1, 0, 1], [0, 2, 1], [9, 0], [0, 6, 0], [0, 0, 0, 1], + [1, 6], [0, 7, 0], +] + + +@dataclass(frozen=True) +class TreeBuffers: + choices: tuple[tuple[int, ...], ...] + tree_indices: Tensor + position_ids: Tensor + retrieve_indices: Tensor + attention_mask: Tensor + + +@dataclass(frozen=True) +class TreeProposal: + bonus_token: Tensor + candidates: Tensor + lengths: Tensor + medusa_logits: Tensor + + +@dataclass(frozen=True) +class Acceptance: + path_index: int + accept_length: int + accepted_tokens: Tensor + + +def _validate_choices(medusa_choices: Sequence[Sequence[int]], top_k: int) -> list[tuple[int, ...]]: + if top_k < 1: + raise ValueError("top_k must be a positive integer") + choices = [tuple(int(value) for value in path) for path in medusa_choices] + if not choices: + raise ValueError("medusa_choices must not be empty") + if len(set(choices)) != len(choices): + raise ValueError("medusa_choices must not contain duplicates") + available = set(choices) + for path in choices: + if not path: + raise ValueError("medusa_choices paths must be non-empty") + if any(value < 0 or value >= top_k for value in path): + raise ValueError(f"medusa_choices values must be in [0, {top_k})") + for depth in range(1, len(path)): + if path[:depth] not in available: + raise ValueError(f"medusa_choices must be prefix-closed: {path}") + return choices + + +def generate_tree_buffers( + medusa_choices: Sequence[Sequence[int]] | None = None, + *, + top_k: int = DEFAULT_TOPK, + device: Any = "cpu", +) -> TreeBuffers: + """Port of the reference ``generate_medusa_buffers`` with an explicit top-k.""" + choices = _validate_choices(DEFAULT_MEDUSA_CHOICES if medusa_choices is None else medusa_choices, top_k) + sorted_choices = sorted(choices, key=lambda path: (len(path), path)) + depth_counts: list[int] = [] + previous_depth = 0 + for path in sorted_choices: + depth = len(path) + if depth != previous_depth: + depth_counts.append(0) + depth_counts[depth - 1] += 1 + previous_depth = depth + + tree_len = len(sorted_choices) + 1 + attention_mask = torch.eye(tree_len, dtype=torch.bool) + attention_mask[:, 0] = True + start = 0 + for depth, count in enumerate(depth_counts, start=1): + for offset in range(count): + choice = sorted_choices[start + offset] + if len(choice) > 1: + ancestors = [ + sorted_choices.index(choice[:prefix]) + 1 + for prefix in range(1, len(choice)) + ] + attention_mask[start + offset + 1, ancestors] = True + start += count + + tree_indices = torch.zeros(tree_len, dtype=torch.long) + start = 0 + for depth, count in enumerate(depth_counts): + for offset in range(count): + tree_indices[start + offset + 1] = ( + sorted_choices[start + offset][-1] + top_k * depth + 1) + start += count + + position_ids = torch.zeros(tree_len, dtype=torch.long) + start = 0 + for depth, count in enumerate(depth_counts): + position_ids[start + 1:start + count + 1] = depth + 1 + start += count + + retrieve_nest: list[list[int]] = [] + retrieve_paths: set[tuple[int, ...]] = set() + for index in range(len(sorted_choices)): + choice = sorted_choices[-index - 1] + if choice in retrieve_paths: + continue + retrieve_indices = [] + for depth in range(len(choice)): + prefix = choice[:depth + 1] + retrieve_indices.append(sorted_choices.index(prefix)) + retrieve_paths.add(prefix) + retrieve_nest.append(retrieve_indices) + max_length = max(len(path) for path in retrieve_nest) + padded = [path + [-2] * (max_length - len(path)) for path in retrieve_nest] + retrieve = torch.tensor(padded, dtype=torch.long) + 1 + retrieve = torch.cat([torch.zeros((retrieve.shape[0], 1), dtype=torch.long), retrieve], dim=1) + + return TreeBuffers( + choices=tuple(sorted_choices), + tree_indices=tree_indices.to(device), + position_ids=position_ids.to(device), + retrieve_indices=retrieve.to(device), + attention_mask=attention_mask.unsqueeze(0).unsqueeze(0).to(device), + ) + + +def _draft_dtype(model: torch.nn.Module) -> torch.dtype: + parameter = next(model.parameters(), None) + return parameter.dtype if parameter is not None else torch.float32 + + +class MedusaTreeProposer: + """Build a Medusa candidate tree from one Target final hidden state.""" + + def __init__( + self, + draft_model: torch.nn.Module, + *, + medusa_choices: Sequence[Sequence[int]] | None = None, + top_k: int = DEFAULT_TOPK, + device: Any = None, + ): + self.model = draft_model.eval() + heads = getattr(draft_model, "medusa_num_heads", None) + if not isinstance(heads, int) or heads < 1: + raise ValueError("MedusaTreeProposer requires a heads-only Medusa draft") + choices = DEFAULT_MEDUSA_CHOICES if medusa_choices is None else medusa_choices + if device is None: + tensor = next(draft_model.parameters(), None) + if tensor is None: + tensor = next(draft_model.buffers(), None) + device = tensor.device if tensor is not None else "cpu" + self.buffers = generate_tree_buffers( + [path for path in choices if len(path) <= heads], top_k=top_k, + device=device) + if not self.buffers.choices: + raise ValueError("medusa_choices has no path within the draft's head depth") + self.top_k = top_k + + @torch.no_grad() + def propose(self, last_hidden: Tensor, base_logits: Tensor) -> TreeProposal: + hidden = last_hidden.reshape(1, 1, -1).to( + device=self.buffers.tree_indices.device, dtype=_draft_dtype(self.model)) + logits = self.model(hidden) + if logits.ndim != 4 or logits.shape[0] != self.model.medusa_num_heads: + raise ValueError("Medusa draft must return [heads, batch, sequence, vocab] logits") + head_logits = logits[:, 0, 0] + topk = torch.topk(head_logits, self.top_k, dim=-1).indices.reshape(-1) + flat_base = base_logits.reshape(-1) + if flat_base.numel() != logits.shape[-1]: + raise ValueError("Base logits vocabulary differs from the draft head vocabulary") + bonus = flat_base.argmax() + candidates = torch.cat([bonus.reshape(1), topk]) + tree_tokens = candidates[self.buffers.tree_indices] + extended = torch.cat([tree_tokens, torch.zeros(1, dtype=torch.long, + device=tree_tokens.device)]) + cart = extended[self.buffers.retrieve_indices] + lengths = (self.buffers.retrieve_indices[:, 1:] >= 0).sum(dim=1) + 1 + return TreeProposal( + bonus_token=bonus, + candidates=cart, + lengths=lengths, + medusa_logits=head_logits, + ) + + +class GreedyVerifier(Protocol): + """Caller-owned Target forward. Returns post-norm hidden and next-token argmax.""" + + def prefill(self, prefix_ids: Tensor) -> tuple[Tensor, Tensor]: + ... + + def verify(self, prefix_ids: Tensor, candidates: Tensor, lengths: Tensor) -> Tensor: + ... + + +def accept_greedy(proposal: TreeProposal, next_tokens: Tensor) -> Acceptance: + """Reference greedy acceptance over verified path predictions.""" + candidates = proposal.candidates + lengths = proposal.lengths + if next_tokens.shape[0] != candidates.shape[0]: + raise ValueError("Verifier returned a different number of paths") + best_path = 0 + best_length = -1 + for path in range(candidates.shape[0]): + real = int(lengths[path]) + if real < 1: + continue + accepted = 0 + for depth in range(real - 1): + if int(next_tokens[path, depth]) != int(candidates[path, depth + 1]): + break + accepted += 1 + if accepted > best_length: + best_length = accepted + best_path = path + accepted_tokens = candidates[best_path, :best_length + 1].clone() + return Acceptance(path_index=best_path, accept_length=best_length, + accepted_tokens=accepted_tokens) + + +class MedusaConsumer: + """Greedy speculative steps with explicit acceptance accounting.""" + + def __init__( + self, + draft_model: torch.nn.Module, + verifier: GreedyVerifier, + *, + medusa_choices: Sequence[Sequence[int]] | None = None, + top_k: int = DEFAULT_TOPK, + ): + self.proposer = MedusaTreeProposer( + draft_model, medusa_choices=medusa_choices, top_k=top_k) + self.verifier = verifier + + @torch.no_grad() + def step(self, prefix_ids: Tensor) -> Acceptance: + hidden, base_logits = self.verifier.prefill(prefix_ids) + proposal = self.proposer.propose(hidden, base_logits) + next_tokens = self.verifier.verify(prefix_ids, proposal.candidates, proposal.lengths) + return accept_greedy(proposal, next_tokens) + + @torch.no_grad() + def generate( + self, + prefix_ids: Tensor, + *, + max_new_tokens: int, + eos_token_id: int | None = None, + ) -> dict[str, Any]: + if max_new_tokens < 1: + raise ValueError("max_new_tokens must be positive") + prefix = prefix_ids.reshape(1, -1).long() + generated = prefix + steps = 0 + accepted_tokens = 0 + while accepted_tokens < max_new_tokens: + acceptance = self.step(generated) + new = acceptance.accepted_tokens + remaining = max_new_tokens - accepted_tokens + if new.numel() > remaining: + new = new[:remaining] + generated = torch.cat([generated, new.reshape(1, -1)], dim=1) + accepted_tokens += new.numel() + steps += 1 + if eos_token_id is not None and int(eos_token_id) in new.tolist(): + stop = generated[0].tolist().index(int(eos_token_id)) + generated = generated[:, :stop + 1] + accepted_tokens = generated.shape[1] - prefix.shape[1] + break + return { + "token_ids": generated[0], + "generated_tokens": accepted_tokens, + "verifier_calls": steps, + "acceptance_length": accepted_tokens / steps if steps else None, + } + + +class HFCausalVerifier: + """Reference verifier: batched teacher-forced paths through a Hugging Face causal LM. + + Exact for greedy acceptance. It re-runs the prefix on every step and makes no + throughput, KV-cache or serving claim. + """ + + def __init__( + self, + model: torch.nn.Module, + *, + device: Any = "cpu", + verify_batch_size: int = 16, + ): + if verify_batch_size < 1: + raise ValueError("verify_batch_size must be positive") + self.model = model.eval() + self.device = torch.device(device) + self.verify_batch_size = verify_batch_size + self.base = getattr(model, "model", None) + if self.base is None or not hasattr(self.base, "forward"): + raise ValueError("HFCausalVerifier requires a causal LM exposing model.model") + self.lm_head = getattr(model, "lm_head", None) + if self.lm_head is None: + raise ValueError("HFCausalVerifier requires a causal LM exposing lm_head") + + def _forward(self, input_ids: Tensor, attention_mask: Tensor) -> Tensor: + outputs = self.base( + input_ids=input_ids.to(self.device), + attention_mask=attention_mask.to(self.device), + use_cache=False, + ) + return outputs.last_hidden_state + + @torch.no_grad() + def prefill(self, prefix_ids: Tensor) -> tuple[Tensor, Tensor]: + prefix = prefix_ids.reshape(1, -1).long() + hidden = self._forward(prefix, torch.ones_like(prefix))[0, -1] + logits = self.lm_head(hidden) + return hidden, logits + + @torch.no_grad() + def verify(self, prefix_ids: Tensor, candidates: Tensor, lengths: Tensor) -> Tensor: + prefix = prefix_ids.reshape(-1).long() + paths = candidates.to(self.device) + lengths = lengths.to(self.device) + width = int(lengths.max()) + outputs = torch.empty( + (paths.shape[0], width - 1), dtype=torch.long, device=self.device) + for start in range(0, paths.shape[0], self.verify_batch_size): + stop = min(start + self.verify_batch_size, paths.shape[0]) + rows = paths[start:stop, :width] + mask = (torch.arange(width, device=self.device)[None, :] + < lengths[start:stop, None]) + sequences = torch.cat( + [prefix[None, :].expand(rows.shape[0], -1), + torch.where(mask, rows, torch.zeros_like(rows))], dim=1) + attention = torch.cat( + [torch.ones((rows.shape[0], prefix.numel()), dtype=torch.long, + device=self.device), mask.long()], dim=1) + hidden = self._forward(sequences, attention) + path_hidden = hidden[:, prefix.numel():prefix.numel() + width - 1] + next_tokens = self.lm_head(path_hidden).argmax(dim=-1) + outputs[start:stop] = next_tokens + return outputs + + @torch.no_grad() + def greedy_generate( + self, + prefix_ids: Tensor, + *, + max_new_tokens: int, + eos_token_id: int | None = None, + ) -> dict[str, Any]: + if max_new_tokens < 1: + raise ValueError("max_new_tokens must be positive") + generated = prefix_ids.reshape(1, -1).long() + steps = 0 + while steps < max_new_tokens: + _, logits = self.prefill(generated) + token = logits.argmax().reshape(1, 1) + generated = torch.cat([generated, token], dim=1) + steps += 1 + if eos_token_id is not None and int(token) == int(eos_token_id): + break + return { + "token_ids": generated[0], + "generated_tokens": steps, + "verifier_calls": steps, + "acceptance_length": 1.0, + } diff --git a/docs/support.md b/docs/support.md index fe9a9bd..7d693e2 100644 --- a/docs/support.md +++ b/docs/support.md @@ -13,7 +13,7 @@ and tests do not imply CPU training support. | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | | Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export, strict reload, proposal consumer and a bounded SpecForge comparison are validated; longer convergence/decoding acceptance pending; see [scope](user-guide/domino.md) | -| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP, warmstart or candidate-tree acceptance yet; see [scope](user-guide/medusa.md) | +| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; greedy candidate-tree acceptance via a standalone Hugging Face evaluator; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP or warmstart; no SGLang Medusa integration, sampling acceptance or production-serving claim; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | | JetSpec | Qwen3-style causal parallel Draft head and temperature-scaled forward-KL objective, reused DFlash block machinery; chunked full-vocabulary KL, explicit causal visibility and exact recovery with FlexAttention, CP1, padded/non-packing | No tree builder, native export or consumer acceptance yet; CPU causal packing and direct unmasked Draft calls fail closed; see [scope](user-guide/jetspec.md) | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index 4865d78..a7c6bba 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -168,5 +168,6 @@ the sum of per-head mean cross-entropies (assistant tokens only): each head's CE within a microbatch, and microbatches are weighted equally by gradient accumulation rather than pooled by token count. The heads own trainable full-vocabulary projections. Start from `configs/recipes/medusa_qwen3_8b.yaml` (SDPA, CP1, MBS1, non-packing, REPLICATE, Store/TCP). -Native export, strict reload and per-head held-out reporting are implemented; candidate-tree -acceptance remains future work; see [details](medusa.md). +Native export, strict reload and per-head held-out reporting are implemented, as is greedy +candidate-tree acceptance through a standalone Hugging Face evaluator; SGLang integration, +sampling acceptance and production serving remain future work; see [details](medusa.md). diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index 4608b29..821317d 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -81,5 +81,21 @@ The report records the artifact and dataset identities, per-head token counts, t mean CE and token accuracy, and the sum of the per-head means. These are held-out training observations, not candidate-tree acceptance. -Candidate-tree acceptance and decode-equality under a Medusa evaluator are not part of this -scope; they require a verified consumer adapter. +## Candidate-tree acceptance + +`aspec.inference.medusa_consumer` builds the reference Medusa tree (one Target bonus token plus +per-head top-k paths), verifies it, and accounts for accepted tokens. Greedy acceptance and +decode equality can be measured against a Hugging Face Target: + +```bash +python -m tools.evaluate_medusa_acceptance \ + --draft outputs/medusa-export --target Qwen/Qwen3-8B \ + --prompts /path/to/prompts.jsonl --output outputs/medusa-acceptance.json +``` + +The report records `acceptance_length = generated_tokens / verifier_calls` (includes the bonus +token), greedy decode equality against Target-only decoding, and repeat stability, together with +the artifact and tree-config hashes. The bundled verifier batches teacher-forced candidate paths +through the Target and re-runs the prefix on every step: it is a reference evaluator, not a +serving runtime, and it makes no throughput or SGLang-compatibility claim. Sampling acceptance +(typical/nucleus) is not implemented; the default is deterministic greedy. diff --git a/tests/cpu/test_medusa_consumer.py b/tests/cpu/test_medusa_consumer.py new file mode 100644 index 0000000..10feb8b --- /dev/null +++ b/tests/cpu/test_medusa_consumer.py @@ -0,0 +1,119 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Medusa candidate-tree consumer: buffers, proposals, verification and acceptance.""" + +import pytest +import torch +from torch import nn +from transformers import LlamaConfig, LlamaForCausalLM + +from aspec.inference.medusa_consumer import ( + HFCausalVerifier, + MedusaConsumer, + TreeProposal, + accept_greedy, + generate_tree_buffers, +) + + +def _tiny_model(): + torch.manual_seed(5) + config = LlamaConfig( + vocab_size=32, hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, max_position_embeddings=128, + pad_token_id=0, + ) + return LlamaForCausalLM(config).eval() + + +class _FixedDraft(nn.Module): + def __init__(self, heads, vocab): + super().__init__() + torch.manual_seed(3) + self.register_buffer("logits", torch.randn(heads, vocab)) + self.medusa_num_heads = heads + + def forward(self, hidden): + return self.logits[:, None, None, :].expand( + -1, hidden.shape[0], hidden.shape[1], -1) + + +def test_tree_buffers_follow_the_reference_layout(): + buffers = generate_tree_buffers([[0], [1], [0, 0]], top_k=2) + assert buffers.choices == ((0,), (1,), (0, 0)) + assert buffers.tree_indices.tolist() == [0, 1, 2, 3] + assert buffers.position_ids.tolist() == [0, 1, 1, 2] + # The reference emits one retrieval row per maximal path: (0,0) and (1,). + assert buffers.retrieve_indices.shape == (2, 3) + assert buffers.retrieve_indices[:, 0].tolist() == [0, 0] + # Tree position 3 is (0,0); it attends to its ancestor (0,) at position 1. + assert buffers.attention_mask[0, 0, 3, 1].item() is True + assert buffers.attention_mask[0, 0, 3, 2].item() is False + with pytest.raises(ValueError, match="prefix-closed"): + generate_tree_buffers([[0, 0]], top_k=2) + with pytest.raises(ValueError, match="in \\[0, 2\\)"): + generate_tree_buffers([[2]], top_k=2) + + +def test_greedy_acceptance_chooses_the_longest_path(): + proposal = TreeProposal( + bonus_token=torch.tensor(10), + candidates=torch.tensor([[10, 1, 2], [10, 3, 4], [11, 5, 0]]), + lengths=torch.tensor([3, 3, 2]), + medusa_logits=torch.zeros(2, 12), + ) + acceptance = accept_greedy(proposal, torch.tensor([[1, 2], [1, 9], [5, 8]])) + assert acceptance.path_index == 0 + assert acceptance.accept_length == 2 + assert acceptance.accepted_tokens.tolist() == [10, 1, 2] + + +def test_hf_verifier_matches_sequential_teacher_forcing(): + model = _tiny_model() + verifier = HFCausalVerifier(model, verify_batch_size=1) + prefix = torch.tensor([[1, 2, 3, 4, 5]]) + candidates = torch.tensor([[6, 7, 8], [9, 10, 0]]) + lengths = torch.tensor([3, 2]) + actual = verifier.verify(prefix, candidates, lengths) + for row, length in enumerate(lengths.tolist()): + sequence = torch.cat([prefix[0], candidates[row, :length]]) + with torch.no_grad(): + hidden = model.model(input_ids=sequence[None]).last_hidden_state + logits = model.lm_head(hidden[0]) + expected = [int(logits[prefix.shape[1] + index].argmax()) + for index in range(length - 1)] + assert actual[row, :len(expected)].tolist() == expected + + +def test_greedy_speculation_matches_target_and_is_stable(): + model = _tiny_model() + verifier = HFCausalVerifier(model) + draft = _FixedDraft(heads=2, vocab=32) + consumer = MedusaConsumer( + draft, verifier, medusa_choices=[[0], [1], [0, 0], [0, 1]], top_k=2) + prefix = torch.tensor([[1, 2, 3, 4]]) + speculative = consumer.generate(prefix, max_new_tokens=6) + baseline = verifier.greedy_generate(prefix, max_new_tokens=6) + assert speculative["token_ids"].tolist() == baseline["token_ids"].tolist() + assert speculative["verifier_calls"] >= 1 + assert speculative["acceptance_length"] >= 1.0 + repeat = consumer.generate(prefix, max_new_tokens=6) + assert repeat["token_ids"].tolist() == speculative["token_ids"].tolist() + +def test_default_choices_filter_to_the_draft_depth(): + from aspec.commands.evaluate_medusa_acceptance import _choices + + choices = _choices(None, 2) + assert choices and max(len(path) for path in choices) <= 2 + with pytest.raises(ValueError, match="head depth"): + _choices(None, 0) + + +def test_default_tree_matches_the_reference_buffers(): + buffers = generate_tree_buffers(top_k=10) + assert buffers.tree_indices.shape == (64,) + assert buffers.tree_indices[:11].tolist() == list(range(11)) + assert buffers.position_ids[:11].tolist() == [0] + [1] * 10 + assert buffers.retrieve_indices.shape == (42, 5) + assert int(buffers.attention_mask.sum()) == 207 diff --git a/tools/evaluate_medusa_acceptance.py b/tools/evaluate_medusa_acceptance.py new file mode 100644 index 0000000..886bf61 --- /dev/null +++ b/tools/evaluate_medusa_acceptance.py @@ -0,0 +1,13 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Compatibility entry point for the installed evaluate_medusa_acceptance command.""" + +import sys +from importlib import import_module + +_command = import_module("aspec.commands.evaluate_medusa_acceptance") +if __name__ == "__main__": + _command.main() +else: + sys.modules[__name__] = _command From 7ed42dd019bb94e23013fb156db61928b4712326 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Fri, 20 Nov 2026 15:11:40 +0800 Subject: [PATCH 2/6] fix(medusa): move consumer prefixes onto the verifier device --- aspec/inference/medusa_consumer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aspec/inference/medusa_consumer.py b/aspec/inference/medusa_consumer.py index 97a9dc3..2ff442c 100644 --- a/aspec/inference/medusa_consumer.py +++ b/aspec/inference/medusa_consumer.py @@ -282,7 +282,8 @@ def generate( ) -> dict[str, Any]: if max_new_tokens < 1: raise ValueError("max_new_tokens must be positive") - prefix = prefix_ids.reshape(1, -1).long() + device = getattr(self.verifier, "device", prefix_ids.device) + prefix = prefix_ids.reshape(1, -1).long().to(device) generated = prefix steps = 0 accepted_tokens = 0 @@ -384,7 +385,7 @@ def greedy_generate( ) -> dict[str, Any]: if max_new_tokens < 1: raise ValueError("max_new_tokens must be positive") - generated = prefix_ids.reshape(1, -1).long() + generated = prefix_ids.reshape(1, -1).long().to(self.device) steps = 0 while steps < max_new_tokens: _, logits = self.prefill(generated) From 86b3239389e4ca1f0320f00864fbd677e2dfe1a4 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Fri, 20 Nov 2026 15:21:40 +0800 Subject: [PATCH 3/6] test(medusa): cover EOS truncation in speculative generation --- tests/cpu/test_medusa_consumer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/cpu/test_medusa_consumer.py b/tests/cpu/test_medusa_consumer.py index 10feb8b..d321923 100644 --- a/tests/cpu/test_medusa_consumer.py +++ b/tests/cpu/test_medusa_consumer.py @@ -117,3 +117,19 @@ def test_default_tree_matches_the_reference_buffers(): assert buffers.position_ids[:11].tolist() == [0] + [1] * 10 assert buffers.retrieve_indices.shape == (42, 5) assert int(buffers.attention_mask.sum()) == 207 + + +def test_eos_stops_both_paths_at_the_same_token(): + model = _tiny_model() + verifier = HFCausalVerifier(model) + draft = _FixedDraft(heads=2, vocab=32) + consumer = MedusaConsumer( + draft, verifier, medusa_choices=[[0], [1], [0, 0], [0, 1]], top_k=2) + prefix = torch.tensor([[1, 2, 3, 4]]) + probe = verifier.greedy_generate(prefix, max_new_tokens=6) + eos = int(probe["token_ids"][prefix.shape[1]]) + baseline = verifier.greedy_generate( + prefix, max_new_tokens=6, eos_token_id=eos) + speculative = consumer.generate(prefix, max_new_tokens=6, eos_token_id=eos) + assert speculative["token_ids"].tolist() == baseline["token_ids"].tolist() + assert speculative["token_ids"][-1].item() == eos From 7ec77217e42f3f144576e7d24ad1de9a38ce14ef Mon Sep 17 00:00:00 2001 From: bohaotu Date: Fri, 20 Nov 2026 15:35:47 +0800 Subject: [PATCH 4/6] feat(medusa): add typical and nucleus acceptance policies Extend the consumer beyond greedy: deterministic typical acceptance and seeded nucleus sampling, with verifier logits, CLI flags, docs and tests. Decode equality stays greedy-only; sampling reports acceptance and repeat stability only. --- aspec/commands/evaluate_medusa_acceptance.py | 60 ++++++- aspec/inference/medusa_consumer.py | 157 ++++++++++++++++++- docs/support.md | 2 +- docs/user-guide/algorithms.md | 6 +- docs/user-guide/medusa.md | 6 +- tests/cpu/test_medusa_consumer.py | 64 ++++++++ 6 files changed, 277 insertions(+), 18 deletions(-) diff --git a/aspec/commands/evaluate_medusa_acceptance.py b/aspec/commands/evaluate_medusa_acceptance.py index dee094c..09dabe6 100644 --- a/aspec/commands/evaluate_medusa_acceptance.py +++ b/aspec/commands/evaluate_medusa_acceptance.py @@ -37,6 +37,15 @@ def _parse_args(): parser.add_argument("--choices", type=Path, default=None) parser.add_argument("--top-k", type=int, default=10) parser.add_argument("--verify-batch-size", type=int, default=16) + parser.add_argument("--sampling", choices=("greedy", "typical", "nucleus"), + default="greedy") + parser.add_argument("--temperature", type=float, default=0.7) + parser.add_argument("--top-p", type=float, default=0.8, dest="top_p") + parser.add_argument("--posterior-threshold", type=float, default=0.3, + dest="posterior_threshold") + parser.add_argument("--posterior-alpha", type=float, default=0.09, + dest="posterior_alpha") + parser.add_argument("--seed", type=int, default=0) parser.add_argument("--device", default=None) parser.add_argument( "--dtype", choices=("bfloat16", "float16", "float32"), default="bfloat16") @@ -70,17 +79,46 @@ def main(): args.prompts, tokenizer, args.limit, enable_thinking=True if args.enable_thinking else None) + greedy = args.sampling == "greedy" + if greedy: + temperature = 0.0 + else: + if args.temperature <= 0: + raise ValueError("sampling acceptance requires --temperature > 0") + temperature = args.temperature + + def generator(): + value = torch.Generator(device=torch.device(device)) + value.manual_seed(args.seed) + return value + eos = tokenizer.eos_token_id results = [] for ids in prompts: prefix = torch.tensor(ids, dtype=torch.long) - baseline = verifier.greedy_generate( - prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos) + if greedy: + baseline = verifier.greedy_generate( + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos) + else: + baseline = None speculative = consumer.generate( - prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos) + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos, + sampling=args.sampling, temperature=temperature, top_p=args.top_p, + posterior_threshold=args.posterior_threshold, + posterior_alpha=args.posterior_alpha, + generator=None if args.sampling != "nucleus" else generator()) repeat = consumer.generate( - prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos) + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos, + sampling=args.sampling, temperature=temperature, top_p=args.top_p, + posterior_threshold=args.posterior_threshold, + posterior_alpha=args.posterior_alpha, + generator=None if args.sampling != "nucleus" else generator()) start = len(ids) + equal = None + if baseline is not None: + equal = ( + speculative["token_ids"][start:].tolist() + == baseline["token_ids"][start:].tolist()) results.append({ "prompt_sha256": hashlib.sha256( json.dumps(ids).encode("utf-8")).hexdigest(), @@ -88,9 +126,7 @@ def main(): "generated_tokens": speculative["generated_tokens"], "verifier_calls": speculative["verifier_calls"], "acceptance_length": speculative["acceptance_length"], - "decode_equal": ( - speculative["token_ids"][start:].tolist() - == baseline["token_ids"][start:].tolist()), + "decode_equal": equal, "repeat_stable": ( speculative["token_ids"].tolist() == repeat["token_ids"].tolist()), }) @@ -109,6 +145,12 @@ def main(): "prompt_count": len(results), "max_new_tokens": args.max_new_tokens, "top_k": args.top_k, + "sampling": args.sampling, + "temperature": temperature, + "top_p": args.top_p if not greedy else None, + "posterior_threshold": (args.posterior_threshold if not greedy else None), + "posterior_alpha": args.posterior_alpha if not greedy else None, + "seed": args.seed if args.sampling == "nucleus" else None, "choices_sha256": hashlib.sha256(choices_blob.encode("utf-8")).hexdigest(), "choices_count": len(choices), "device": device, @@ -119,7 +161,9 @@ def main(): "acceptance_length": total_tokens / total_calls, "acceptance_convention": ( "generated_tokens / verifier_calls; includes the bonus token"), - "decode_equal": all(item["decode_equal"] for item in results), + "decode_equal": (all(item["decode_equal"] for item in results) + if greedy else None), + "decode_equal_applicable": greedy, "repeat_stable": all(item["repeat_stable"] for item in results), "results": results, } diff --git a/aspec/inference/medusa_consumer.py b/aspec/inference/medusa_consumer.py index 2ff442c..68165bf 100644 --- a/aspec/inference/medusa_consumer.py +++ b/aspec/inference/medusa_consumer.py @@ -225,6 +225,84 @@ def verify(self, prefix_ids: Tensor, candidates: Tensor, lengths: Tensor) -> Ten ... +def _valid_steps(proposal: TreeProposal, width: int) -> Tensor: + steps = torch.arange(width - 1, device=proposal.candidates.device)[None, :] + return steps < (proposal.lengths - 1)[:, None] + + +def _pick_best(proposal: TreeProposal, mask: Tensor, *, likelihood: Tensor | None = None) -> Acceptance: + lengths = torch.cumprod(mask, dim=1).sum(dim=1) + best_length = int(lengths.max()) + candidates = (lengths == best_length).nonzero().flatten() + if likelihood is not None and best_length > 0: + scores = torch.log(likelihood[candidates, :best_length] + 1e-12).sum(dim=-1) + best = candidates[scores.argmax()] + else: + best = candidates[0] + accepted = proposal.candidates[best, :best_length + 1].clone() + return Acceptance(path_index=int(best), accept_length=int(best_length), + accepted_tokens=accepted) + + +def accept_typical( + proposal: TreeProposal, + logits: Tensor, + *, + temperature: float, + posterior_threshold: float = 0.3, + posterior_alpha: float = 0.09, +) -> Acceptance: + """Reference typical-acceptance fast path (deterministic).""" + if temperature <= 0: + raise ValueError("typical acceptance requires temperature > 0") + if not 0 <= posterior_threshold <= 1: + raise ValueError("posterior_threshold must be in [0, 1]") + if posterior_alpha < 0: + raise ValueError("posterior_alpha must be non-negative") + width = proposal.candidates.shape[1] + probs = torch.softmax(logits[:, :width - 1].float() / temperature, dim=-1) + tokens = proposal.candidates[:, 1:width].to(probs.device) + candidate_prob = probs.gather(-1, tokens.unsqueeze(-1)).squeeze(-1) + entropy = -(probs * torch.log(probs + 1e-5)).sum(dim=-1) + threshold = torch.minimum( + torch.full_like(entropy, posterior_threshold), + torch.exp(-entropy) * posterior_alpha, + ) + mask = (candidate_prob > threshold) & _valid_steps(proposal, width).to(probs.device) + return _pick_best(proposal, mask, likelihood=candidate_prob) + + +def accept_nucleus( + proposal: TreeProposal, + logits: Tensor, + *, + temperature: float, + top_p: float, + generator: torch.Generator, +) -> Acceptance: + """Reference nucleus sampling acceptance.""" + if temperature <= 0: + raise ValueError("nucleus acceptance requires temperature > 0") + if not 0 < top_p <= 1: + raise ValueError("top_p must be in (0, 1]") + width = proposal.candidates.shape[1] + probs = torch.softmax(logits[:, :width - 1].float() / temperature, dim=-1) + flat = probs.reshape(-1, probs.shape[-1]) + if top_p < 1: + sorted_probs, sorted_indices = torch.sort(flat, descending=True, dim=-1) + cumulative = torch.cumsum(sorted_probs, dim=-1) + remove = cumulative > top_p + remove[..., 1:] = remove[..., :-1].clone() + remove[..., 0] = False + filtered = torch.zeros_like(remove).scatter(1, sorted_indices, remove) + flat = flat.masked_fill(filtered, 0.0) + flat = flat / flat.sum(dim=-1, keepdim=True).clamp_min(1e-12) + sampled = torch.multinomial(flat, 1, generator=generator).view(-1, width - 1) + tokens = proposal.candidates[:, 1:width].to(probs.device) + mask = (tokens == sampled) & _valid_steps(proposal, width).to(probs.device) + return _pick_best(proposal, mask) + + def accept_greedy(proposal: TreeProposal, next_tokens: Tensor) -> Acceptance: """Reference greedy acceptance over verified path predictions.""" candidates = proposal.candidates @@ -266,11 +344,44 @@ def __init__( self.verifier = verifier @torch.no_grad() - def step(self, prefix_ids: Tensor) -> Acceptance: + def step( + self, + prefix_ids: Tensor, + *, + sampling: str = "greedy", + temperature: float = 0.0, + top_p: float = 0.8, + posterior_threshold: float = 0.3, + posterior_alpha: float = 0.09, + generator: torch.Generator | None = None, + ) -> Acceptance: hidden, base_logits = self.verifier.prefill(prefix_ids) proposal = self.proposer.propose(hidden, base_logits) - next_tokens = self.verifier.verify(prefix_ids, proposal.candidates, proposal.lengths) - return accept_greedy(proposal, next_tokens) + if sampling == "greedy": + next_tokens = self.verifier.verify( + prefix_ids, proposal.candidates, proposal.lengths) + return accept_greedy(proposal, next_tokens) + if not hasattr(self.verifier, "verify_logits"): + raise ValueError(f"Verifier does not implement {sampling} acceptance") + logits = self.verifier.verify_logits( + prefix_ids, proposal.candidates, proposal.lengths) + if sampling == "typical": + return accept_typical( + proposal, logits, temperature=temperature, + posterior_threshold=posterior_threshold, + posterior_alpha=posterior_alpha) + if sampling == "nucleus": + if generator is None: + raise ValueError("nucleus acceptance requires a torch.Generator") + if generator.device.type != logits.device.type or ( + generator.device.index is not None + and logits.device.index is not None + and generator.device.index != logits.device.index): + raise ValueError("nucleus generator must be on the logits device") + return accept_nucleus( + proposal, logits, temperature=temperature, top_p=top_p, + generator=generator) + raise ValueError(f"Unsupported sampling policy: {sampling!r}") @torch.no_grad() def generate( @@ -279,6 +390,12 @@ def generate( *, max_new_tokens: int, eos_token_id: int | None = None, + sampling: str = "greedy", + temperature: float = 0.0, + top_p: float = 0.8, + posterior_threshold: float = 0.3, + posterior_alpha: float = 0.09, + generator: torch.Generator | None = None, ) -> dict[str, Any]: if max_new_tokens < 1: raise ValueError("max_new_tokens must be positive") @@ -288,7 +405,10 @@ def generate( steps = 0 accepted_tokens = 0 while accepted_tokens < max_new_tokens: - acceptance = self.step(generated) + acceptance = self.step( + generated, sampling=sampling, temperature=temperature, + top_p=top_p, posterior_threshold=posterior_threshold, + posterior_alpha=posterior_alpha, generator=generator) new = acceptance.accepted_tokens remaining = max_new_tokens - accepted_tokens if new.numel() > remaining: @@ -375,6 +495,35 @@ def verify(self, prefix_ids: Tensor, candidates: Tensor, lengths: Tensor) -> Ten outputs[start:stop] = next_tokens return outputs + @torch.no_grad() + def verify_logits( + self, prefix_ids: Tensor, candidates: Tensor, lengths: Tensor) -> Tensor: + """Return next-token logits at candidate prediction positions [paths, width-1, vocab].""" + prefix = prefix_ids.reshape(-1).long() + paths = candidates.to(self.device) + lengths = lengths.to(self.device) + width = int(lengths.max()) + vocab = getattr(self.lm_head, "out_features", None) + if vocab is None: + vocab = int(self.model.config.vocab_size) + outputs = torch.empty( + (paths.shape[0], width - 1, vocab), dtype=torch.float32, device=self.device) + for start in range(0, paths.shape[0], self.verify_batch_size): + stop = min(start + self.verify_batch_size, paths.shape[0]) + rows = paths[start:stop, :width] + mask = (torch.arange(width, device=self.device)[None, :] + < lengths[start:stop, None]) + sequences = torch.cat( + [prefix[None, :].expand(rows.shape[0], -1), + torch.where(mask, rows, torch.zeros_like(rows))], dim=1) + attention = torch.cat( + [torch.ones((rows.shape[0], prefix.numel()), dtype=torch.long, + device=self.device), mask.long()], dim=1) + hidden = self._forward(sequences, attention) + path_hidden = hidden[:, prefix.numel():prefix.numel() + width - 1] + outputs[start:stop] = self.lm_head(path_hidden).float() + return outputs + @torch.no_grad() def greedy_generate( self, diff --git a/docs/support.md b/docs/support.md index 7d693e2..c0d78c0 100644 --- a/docs/support.md +++ b/docs/support.md @@ -13,7 +13,7 @@ and tests do not imply CPU training support. | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | | Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export, strict reload, proposal consumer and a bounded SpecForge comparison are validated; longer convergence/decoding acceptance pending; see [scope](user-guide/domino.md) | -| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; greedy candidate-tree acceptance via a standalone Hugging Face evaluator; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP or warmstart; no SGLang Medusa integration, sampling acceptance or production-serving claim; see [scope](user-guide/medusa.md) | +| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; greedy and typical/nucleus sampling candidate-tree acceptance via a standalone Hugging Face evaluator; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP or warmstart; no SGLang Medusa integration or production-serving claim; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | | JetSpec | Qwen3-style causal parallel Draft head and temperature-scaled forward-KL objective, reused DFlash block machinery; chunked full-vocabulary KL, explicit causal visibility and exact recovery with FlexAttention, CP1, padded/non-packing | No tree builder, native export or consumer acceptance yet; CPU causal packing and direct unmasked Draft calls fail closed; see [scope](user-guide/jetspec.md) | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index a7c6bba..2219ace 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -168,6 +168,6 @@ the sum of per-head mean cross-entropies (assistant tokens only): each head's CE within a microbatch, and microbatches are weighted equally by gradient accumulation rather than pooled by token count. The heads own trainable full-vocabulary projections. Start from `configs/recipes/medusa_qwen3_8b.yaml` (SDPA, CP1, MBS1, non-packing, REPLICATE, Store/TCP). -Native export, strict reload and per-head held-out reporting are implemented, as is greedy -candidate-tree acceptance through a standalone Hugging Face evaluator; SGLang integration, -sampling acceptance and production serving remain future work; see [details](medusa.md). +Native export, strict reload and per-head held-out reporting are implemented, as is greedy and +typical/nucleus sampling candidate-tree acceptance through a standalone Hugging Face evaluator; +SGLang integration and production serving remain future work; see [details](medusa.md). diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index 821317d..f47d791 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -97,5 +97,7 @@ The report records `acceptance_length = generated_tokens / verifier_calls` (incl token), greedy decode equality against Target-only decoding, and repeat stability, together with the artifact and tree-config hashes. The bundled verifier batches teacher-forced candidate paths through the Target and re-runs the prefix on every step: it is a reference evaluator, not a -serving runtime, and it makes no throughput or SGLang-compatibility claim. Sampling acceptance -(typical/nucleus) is not implemented; the default is deterministic greedy. +serving runtime, and it makes no throughput or SGLang-compatibility claim. `--sampling` accepts +`greedy` (default), `typical` (fast deterministic threshold) or `nucleus` (seeded top-p); +`decode_equal` is only reported for greedy, while sampling reports acceptance length and +repeat stability with the same seed. diff --git a/tests/cpu/test_medusa_consumer.py b/tests/cpu/test_medusa_consumer.py index d321923..dbdebca 100644 --- a/tests/cpu/test_medusa_consumer.py +++ b/tests/cpu/test_medusa_consumer.py @@ -13,6 +13,8 @@ MedusaConsumer, TreeProposal, accept_greedy, + accept_nucleus, + accept_typical, generate_tree_buffers, ) @@ -133,3 +135,65 @@ def test_eos_stops_both_paths_at_the_same_token(): speculative = consumer.generate(prefix, max_new_tokens=6, eos_token_id=eos) assert speculative["token_ids"].tolist() == baseline["token_ids"].tolist() assert speculative["token_ids"][-1].item() == eos + + + +def _single_path_proposal(): + return TreeProposal( + bonus_token=torch.tensor(10), + candidates=torch.tensor([[10, 1, 2]]), + lengths=torch.tensor([3]), + medusa_logits=torch.zeros(2, 5), + ) + + +def test_typical_acceptance_stops_at_the_first_low_probability_token(): + logits = torch.zeros(1, 2, 5) + logits[0, 0, 1] = 5.0 + logits[0, 1, 3] = 5.0 + acceptance = accept_typical( + _single_path_proposal(), logits, temperature=1.0, + posterior_threshold=0.3, posterior_alpha=0.09) + assert acceptance.accept_length == 1 + assert acceptance.accepted_tokens.tolist() == [10, 1] + + +def test_nucleus_acceptance_is_seeded_and_masks_mismatches(): + logits = torch.zeros(1, 2, 5) + logits[0, 0, 1] = 5.0 + logits[0, 1, 3] = 5.0 + first = accept_nucleus( + _single_path_proposal(), logits, temperature=1.0, top_p=1.0, + generator=torch.Generator().manual_seed(7)) + second = accept_nucleus( + _single_path_proposal(), logits, temperature=1.0, top_p=1.0, + generator=torch.Generator().manual_seed(7)) + assert first.accept_length == 1 + assert first.accepted_tokens.tolist() == [10, 1] + assert second.accepted_tokens.tolist() == first.accepted_tokens.tolist() + + +def test_verify_logits_argmax_matches_verify(): + model = _tiny_model() + verifier = HFCausalVerifier(model) + prefix = torch.tensor([[1, 2, 3, 4]]) + candidates = torch.tensor([[6, 7, 8], [9, 10, 0]]) + lengths = torch.tensor([3, 2]) + logits = verifier.verify_logits(prefix, candidates, lengths) + assert logits.shape == (2, 2, 32) + assert torch.equal(logits.argmax(-1), verifier.verify(prefix, candidates, lengths)) + + +def test_sampling_generation_is_repeatable(): + model = _tiny_model() + verifier = HFCausalVerifier(model) + draft = _FixedDraft(heads=2, vocab=32) + consumer = MedusaConsumer( + draft, verifier, medusa_choices=[[0], [1], [0, 0], [0, 1]], top_k=2) + prefix = torch.tensor([[1, 2, 3, 4]]) + first = consumer.generate( + prefix, max_new_tokens=3, sampling="typical", temperature=1.0) + second = consumer.generate( + prefix, max_new_tokens=3, sampling="typical", temperature=1.0) + assert first["token_ids"].tolist() == second["token_ids"].tolist() + assert first["verifier_calls"] >= 1 From f36c1e0b02d7fa2ce1ddf383e4d1df781779b139 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Sat, 21 Nov 2026 22:28:28 +0800 Subject: [PATCH 5/6] fix(medusa): sample nucleus bonus and scope EOS truncation Nucleus acceptance now draws the bonus token from the top-p base distribution with the supplied generator instead of fixing it by argmax, and EOS handling searches only the newly accepted tokens so an EOS already in the prefix cannot truncate the prompt or produce a negative token count. --- aspec/inference/medusa_consumer.py | 68 ++++++++++++++++++++++++------ tests/cpu/test_medusa_consumer.py | 51 ++++++++++++++++++++++ 2 files changed, 105 insertions(+), 14 deletions(-) diff --git a/aspec/inference/medusa_consumer.py b/aspec/inference/medusa_consumer.py index 68165bf..b67d035 100644 --- a/aspec/inference/medusa_consumer.py +++ b/aspec/inference/medusa_consumer.py @@ -189,7 +189,10 @@ def __init__( self.top_k = top_k @torch.no_grad() - def propose(self, last_hidden: Tensor, base_logits: Tensor) -> TreeProposal: + def propose( + self, last_hidden: Tensor, base_logits: Tensor, *, + bonus_token: Tensor | None = None, + ) -> TreeProposal: hidden = last_hidden.reshape(1, 1, -1).to( device=self.buffers.tree_indices.device, dtype=_draft_dtype(self.model)) logits = self.model(hidden) @@ -200,7 +203,12 @@ def propose(self, last_hidden: Tensor, base_logits: Tensor) -> TreeProposal: flat_base = base_logits.reshape(-1) if flat_base.numel() != logits.shape[-1]: raise ValueError("Base logits vocabulary differs from the draft head vocabulary") - bonus = flat_base.argmax() + if bonus_token is None: + bonus = flat_base.argmax() + else: + bonus = bonus_token.reshape(-1).to(device=flat_base.device, dtype=torch.long) + if bonus.numel() != 1: + raise ValueError("bonus_token must be a scalar") candidates = torch.cat([bonus.reshape(1), topk]) tree_tokens = candidates[self.buffers.tree_indices] extended = torch.cat([tree_tokens, torch.zeros(1, dtype=torch.long, @@ -272,6 +280,35 @@ def accept_typical( return _pick_best(proposal, mask, likelihood=candidate_prob) +def _nucleus_probs(logits: Tensor, *, temperature: float, top_p: float) -> Tensor: + probs = torch.softmax(logits.float() / temperature, dim=-1) + if top_p < 1: + sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1) + cumulative = torch.cumsum(sorted_probs, dim=-1) + remove = cumulative > top_p + remove[..., 1:] = remove[..., :-1].clone() + remove[..., 0] = False + filtered = torch.zeros_like(remove).scatter(1, sorted_indices, remove) + probs = probs.masked_fill(filtered, 0.0) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp_min(1e-12) + return probs + + +def sample_nucleus_token( + logits: Tensor, *, temperature: float, top_p: float, + generator: torch.Generator, +) -> Tensor: + """Draw the bonus token from the nucleus distribution, as the reference does.""" + if temperature <= 0: + raise ValueError("nucleus sampling requires temperature > 0") + if not 0 < top_p <= 1: + raise ValueError("top_p must be in (0, 1]") + if generator.device.type != logits.device.type: + raise ValueError("nucleus generator must be on the logits device") + probs = _nucleus_probs(logits.reshape(1, -1), temperature=temperature, top_p=top_p) + return torch.multinomial(probs, 1, generator=generator).reshape(()) + + def accept_nucleus( proposal: TreeProposal, logits: Tensor, @@ -286,17 +323,9 @@ def accept_nucleus( if not 0 < top_p <= 1: raise ValueError("top_p must be in (0, 1]") width = proposal.candidates.shape[1] - probs = torch.softmax(logits[:, :width - 1].float() / temperature, dim=-1) + probs = _nucleus_probs( + logits[:, :width - 1], temperature=temperature, top_p=top_p) flat = probs.reshape(-1, probs.shape[-1]) - if top_p < 1: - sorted_probs, sorted_indices = torch.sort(flat, descending=True, dim=-1) - cumulative = torch.cumsum(sorted_probs, dim=-1) - remove = cumulative > top_p - remove[..., 1:] = remove[..., :-1].clone() - remove[..., 0] = False - filtered = torch.zeros_like(remove).scatter(1, sorted_indices, remove) - flat = flat.masked_fill(filtered, 0.0) - flat = flat / flat.sum(dim=-1, keepdim=True).clamp_min(1e-12) sampled = torch.multinomial(flat, 1, generator=generator).view(-1, width - 1) tokens = proposal.candidates[:, 1:width].to(probs.device) mask = (tokens == sampled) & _valid_steps(proposal, width).to(probs.device) @@ -356,7 +385,17 @@ def step( generator: torch.Generator | None = None, ) -> Acceptance: hidden, base_logits = self.verifier.prefill(prefix_ids) - proposal = self.proposer.propose(hidden, base_logits) + if sampling == "nucleus": + if generator is None: + raise ValueError("nucleus acceptance requires a torch.Generator") + if generator.device.type != base_logits.device.type: + raise ValueError("nucleus generator must be on the logits device") + bonus = sample_nucleus_token( + base_logits, temperature=temperature, top_p=top_p, + generator=generator) + proposal = self.proposer.propose(hidden, base_logits, bonus_token=bonus) + else: + proposal = self.proposer.propose(hidden, base_logits) if sampling == "greedy": next_tokens = self.verifier.verify( prefix_ids, proposal.candidates, proposal.lengths) @@ -417,7 +456,8 @@ def generate( accepted_tokens += new.numel() steps += 1 if eos_token_id is not None and int(eos_token_id) in new.tolist(): - stop = generated[0].tolist().index(int(eos_token_id)) + stop = (generated.shape[1] - new.numel() + + new.tolist().index(int(eos_token_id))) generated = generated[:, :stop + 1] accepted_tokens = generated.shape[1] - prefix.shape[1] break diff --git a/tests/cpu/test_medusa_consumer.py b/tests/cpu/test_medusa_consumer.py index dbdebca..89548d4 100644 --- a/tests/cpu/test_medusa_consumer.py +++ b/tests/cpu/test_medusa_consumer.py @@ -9,6 +9,7 @@ from transformers import LlamaConfig, LlamaForCausalLM from aspec.inference.medusa_consumer import ( + Acceptance, HFCausalVerifier, MedusaConsumer, TreeProposal, @@ -16,6 +17,7 @@ accept_nucleus, accept_typical, generate_tree_buffers, + sample_nucleus_token, ) @@ -197,3 +199,52 @@ def test_sampling_generation_is_repeatable(): prefix, max_new_tokens=3, sampling="typical", temperature=1.0) assert first["token_ids"].tolist() == second["token_ids"].tolist() assert first["verifier_calls"] >= 1 + + + +def test_nucleus_bonus_is_sampled_from_the_base_distribution(): + logits = torch.log(torch.tensor([[0.6, 0.4]])) + picks = { + int(sample_nucleus_token( + logits, temperature=1.0, top_p=1.0, + generator=torch.Generator().manual_seed(seed))) + for seed in range(20) + } + assert picks == {0, 1} + + +def test_nucleus_step_uses_the_sampled_bonus(): + class Verifier: + device = torch.device("cpu") + + def prefill(self, ids): + return torch.zeros(1), torch.log(torch.tensor([0.6, 0.4])) + + def verify_logits(self, prefix, candidates, lengths): + width = int(lengths.max()) + return torch.zeros((candidates.shape[0], width - 1, 2)) + + draft = _FixedDraft(heads=2, vocab=2) + consumer = MedusaConsumer( + draft, Verifier(), medusa_choices=[[0], [1]], top_k=2) + base = torch.log(torch.tensor([0.6, 0.4])) + expected = int(sample_nucleus_token( + base, temperature=1.0, top_p=1.0, + generator=torch.Generator().manual_seed(3))) + acceptance = consumer.step( + torch.tensor([[1]]), sampling="nucleus", temperature=1.0, top_p=1.0, + generator=torch.Generator().manual_seed(3)) + assert int(acceptance.accepted_tokens[0]) == expected + + +def test_eos_is_searched_only_in_new_tokens(): + from types import SimpleNamespace + + consumer = object.__new__(MedusaConsumer) + consumer.verifier = SimpleNamespace(device=torch.device("cpu")) + consumer.step = lambda ids, **kwargs: Acceptance( + path_index=0, accept_length=1, accepted_tokens=torch.tensor([9, 2])) + result = consumer.generate( + torch.tensor([[7, 2, 8]]), max_new_tokens=5, eos_token_id=2) + assert result["token_ids"].tolist() == [7, 2, 8, 9, 2] + assert result["generated_tokens"] == 2 From 069593b9c87040bf2ef002a047e502d4701e26a5 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Sat, 21 Nov 2026 23:18:55 +0800 Subject: [PATCH 6/6] fix(medusa): flatten nucleus logits before top-p filtering Top-p filtering indexed the vocabulary axis while the tensor still carried the path/position axes, which triggers a device-side assert on CUDA. Flatten the prediction positions first and add a multi-position top-p regression test. --- aspec/inference/medusa_consumer.py | 7 +++---- tests/cpu/test_medusa_consumer.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/aspec/inference/medusa_consumer.py b/aspec/inference/medusa_consumer.py index b67d035..6182962 100644 --- a/aspec/inference/medusa_consumer.py +++ b/aspec/inference/medusa_consumer.py @@ -323,10 +323,9 @@ def accept_nucleus( if not 0 < top_p <= 1: raise ValueError("top_p must be in (0, 1]") width = proposal.candidates.shape[1] - probs = _nucleus_probs( - logits[:, :width - 1], temperature=temperature, top_p=top_p) - flat = probs.reshape(-1, probs.shape[-1]) - sampled = torch.multinomial(flat, 1, generator=generator).view(-1, width - 1) + flat_logits = logits[:, :width - 1].reshape(-1, logits.shape[-1]) + probs = _nucleus_probs(flat_logits, temperature=temperature, top_p=top_p) + sampled = torch.multinomial(probs, 1, generator=generator).view(-1, width - 1) tokens = proposal.candidates[:, 1:width].to(probs.device) mask = (tokens == sampled) & _valid_steps(proposal, width).to(probs.device) return _pick_best(proposal, mask) diff --git a/tests/cpu/test_medusa_consumer.py b/tests/cpu/test_medusa_consumer.py index 89548d4..0ef5d25 100644 --- a/tests/cpu/test_medusa_consumer.py +++ b/tests/cpu/test_medusa_consumer.py @@ -248,3 +248,15 @@ def test_eos_is_searched_only_in_new_tokens(): torch.tensor([[7, 2, 8]]), max_new_tokens=5, eos_token_id=2) assert result["token_ids"].tolist() == [7, 2, 8, 9, 2] assert result["generated_tokens"] == 2 + + + +def test_nucleus_acceptance_filters_top_p_across_positions(): + logits = torch.zeros(1, 2, 5) + logits[0, 0, 1] = 5.0 + logits[0, 1, 3] = 5.0 + acceptance = accept_nucleus( + _single_path_proposal(), logits, temperature=1.0, top_p=0.8, + generator=torch.Generator().manual_seed(7)) + assert acceptance.accept_length == 1 + assert acceptance.accepted_tokens.tolist() == [10, 1]