From 9ae9363fa4a6a7e5f3b87fd3a5e104d83cd23081 Mon Sep 17 00:00:00 2001 From: AgentDescent Dev-4 Date: Sat, 21 Nov 2026 19:18:27 +0000 Subject: [PATCH 1/3] feat(bench): long-horizon continual run with lineage (#941) Every other bench here measures one hop. RSI is a long loop: the same artifact rewritten step after step, each step conditioned on the version the previous one committed, and the usual failure is a decision that was locally right at step 4 and wrong at step 12. bench/long_horizon.py evolves one playbook across 1-15 steps (default 9) on a Ledger, with a deterministic offline proposer and DefaultAcceptance gating each rewrite on the held-out checks. The six check families conflict by construction (a later rule line overrides an earlier one, so the playbook has a history but cannot hold two answers), which lets the bench separate aggregate retention from per-family forgetting: on the 3-seed run every seed ends at the domain's reachable ceiling of 0.583 and is still verdict `forgetting`, because two families were erased on the way. Metrics are defined once: retention, TaskCurve's forgetting (mean/max/fraction), paired transfer on a fresh check set via bootstrap_ci, and cost. Seeds shuffle only the arrival order; the held-out split is hash-based and reported rather than assumed, and a run whose gate cannot see a family says so. --- bench/long_horizon.py | 676 +++++ bench/results/long-horizon-3seed.json | 3967 +++++++++++++++++++++++++ changelog.d/941-long-horizon-bench.md | 9 + docs/long-horizon-bench.md | 140 + mkdocs.yml | 1 + tests/test_long_horizon_bench.py | 136 + 6 files changed, 4929 insertions(+) create mode 100644 bench/long_horizon.py create mode 100644 bench/results/long-horizon-3seed.json create mode 100644 changelog.d/941-long-horizon-bench.md create mode 100644 docs/long-horizon-bench.md create mode 100644 tests/test_long_horizon_bench.py diff --git a/bench/long_horizon.py b/bench/long_horizon.py new file mode 100644 index 00000000..9cee7351 --- /dev/null +++ b/bench/long_horizon.py @@ -0,0 +1,676 @@ +#!/usr/bin/env python3 +"""Long-horizon (continual) benchmark: one artifact, many steps, with lineage. + +Every other bench in this repository measures one hop: a generation loop over a +fixed task set (``bench.rsi_bench``), or a chain of rungs run once +(``bench.self_evolution``). Recursive self-improvement is a **long** loop: the +same artifact is rewritten step after step, each step conditioned on the version +the previous step committed, and the thing that goes wrong at step 12 is usually +a decision that was locally correct at step 4. This bench measures that +directly -- one playbook, ``--steps`` sequential steps, a git ledger recording +which rewrites were committed and why the others were refused. + + python -m bench.long_horizon # human-readable, 9 steps, 3 seeds + python -m bench.long_horizon --json # machine-readable report + python -m bench.long_horizon --steps 15 --json # the long end of the range + +Why this domain +--------------- +The artifact is a playbook ``SKILL.md`` of rule lines. A later line for the +same rule **overrides** the earlier one, so the playbook can carry a history +without being able to hold two answers to the same question. Six check families +read the rules, and they conflict by construction: ``add_sum`` and +``add_product`` want opposite bodies for the same rule, ``chain_add`` needs +``chain: add_then_mul`` *and* ``add: sum``, ``chain_mul`` needs +``chain: mul_then_add`` *and* ``mul: product``. No assignment of rule bodies +scores every check, so the loop cannot win outright -- it can only trade +families off. That is deliberate, and the JSON reports the reachable ceiling +beside the measured retention so a reader can see how much of the gap is the +loop's fault. + +Each step is: + +1. the next family in a seed-shuffled arrival order becomes the target; +2. a deterministic, offline proposer reads the playbook and emits the one rule + line that fixes that family's checks -- no model call, no network; +3. the candidate is measured on the held-out checks; +4. :class:`~agentdescent.defaults.DefaultAcceptance` gates the rewrite and, when + it passes, the :class:`~agentdescent.Ledger` commits it (content-addressed + lineage); otherwise the step is recorded as refused and the artifact is left + exactly as it was. + +The gate reads the **held-out** checks only, and it never sees more than a third +of them. A rewrite that is free on the held-out set can still erase a family +that is passing, which is the failure mode this bench exists to measure. The +default holds out one check per family; lower ``--holdout-fraction`` and the +report names the families nothing is protecting (``families_without_holdout``) +instead of quietly measuring a split. + +Metrics are each defined once, here, and nowhere else: + +* ``retention`` -- the share of every check the final artifact passes; +* ``forgetting`` -- reused verbatim from :class:`~bench.rsi_bench.TaskCurve` + (``max(0, max(valid) - valid[-1])``) per check, reported as a mean, a maximum + and a fraction, exactly as :class:`~bench.rsi_bench.RSIBenchReport` does; +* ``transfer`` -- paired final-minus-step-0 on a fresh check set that no step + ever measured, bootstrapped per seed with + :func:`~agentdescent.stats_eval.bootstrap_ci`; +* ``cost`` -- check evaluations, wall clock, ledger commits and artifact bytes. + +``--steps`` defaults to 9 and the published range is 5-15; anything from 1 to 15 +is accepted so the test can exercise the same mechanics in three steps. + +Seeds shuffle **which family arrives when** and nothing else; the held-out split +is by stable hash and is the same for every seed. Three seeds give a range, not +significance, and the report says so where the number is printed. Negative +results are published as they fall out: ``flat`` and ``regressing`` are verdicts, +not omissions. + +Everything except ``cost.wall_clock_s`` is reproducible byte for byte; git commit +hashes never enter the report, because the lineage is identified by the content +hash of the playbook at each step instead. +""" + +from __future__ import annotations + +import argparse +import hashlib +import itertools +import json +import random +import sys +import tempfile +import time +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from agentdescent import ( + EvolvingArtifact, + FileTree, + Ledger, + MergeContext, + build_manifest, +) +from agentdescent.defaults import DefaultAcceptance +from agentdescent.filetree import canonical +from agentdescent.stats_eval import bootstrap_ci +from bench.rsi_bench import ( + BenchmarkTask, + RSIBench, + RSIBenchReport, + TaskCurve, +) + +__all__ = ["run_bench", "run_seed", "main", "check_tasks", "fresh_tasks"] + +BENCH = "long_horizon" +VERSION = 1 +PLAYBOOK = "SKILL.md" + +#: The playbook's header. No ``- `` prefix, so it never parses as a rule. +HEADER = ( + "# Playbook\n" + "Rules the loop edits. The last line for a rule is the one in force:\n" + "earlier lines stay in the history and are never applied.\n" +) + +#: ``family -> the (rule, body) pairs its checks need``. A check scores the +#: share of its pairs that are in force, so a chain check whose order is right +#: but whose dependency was rewritten away scores one half. +FAMILIES: Dict[str, Tuple[Tuple[str, str], ...]] = { + "add_sum": (("add", "sum"),), + "add_product": (("add", "product"),), + "mul_sum": (("mul", "sum"),), + "mul_product": (("mul", "product"),), + "chain_add": (("chain", "add_then_mul"), ("add", "sum")), + "chain_mul": (("chain", "mul_then_add"), ("mul", "product")), +} +CHECKS_PER_FAMILY = 3 +FRESH_PER_FAMILY = 2 + +#: Every assignment of rule bodies the playbook can express (last line wins, so +#: there is exactly one body in force per rule). +ASSIGNMENTS = tuple( + dict(pairs) + for pairs in itertools.product( + (("add", "sum"), ("add", "product")), + (("mul", "sum"), ("mul", "product")), + (("chain", "add_then_mul"), ("chain", "mul_then_add")), + ) +) + +#: A third of the checks: on the 18-check set this lands exactly one held-out +#: check per family, so the gate can see every family. The composition is +#: recomputed and reported (``checks.holdout_by_family``), never assumed -- a +#: gate that cannot see a family measures the split, not the loop, and +#: ``--holdout-fraction`` below this is what the forgetting test uses. +HOLDOUT_FRACTION = 1 / 3 +GATE = dict(base_delta=0.5, anneal_half_life=64, accept_samples=4000) +MAX_STEPS = 15 + + +# -- the artifact ------------------------------------------------------------- + + +def parse_playbook(text: str) -> Dict[str, str]: + """The rules in force: the **last** line for a rule wins.""" + effective: Dict[str, str] = {} + for line in text.splitlines(): + line = line.strip() + if not line.startswith("- ") or ":" not in line: + continue + rule, body = line[2:].split(":", 1) + rule, body = rule.strip(), body.strip() + if rule and body: + effective[rule] = body + return effective + + +def append_rule(text: str, rule: str, body: str) -> str: + return f"{text.rstrip()}\n- {rule}: {body}\n" + + +def content_sha(state: Mapping[str, str]) -> str: + """A short, stable id for exactly what the playbook says at one step.""" + return hashlib.sha256(canonical(state).encode("utf-8")).hexdigest()[:12] + + +def artifact_id(state: Mapping[str, str]) -> str: + """``filetree.canonical(state)`` is the id, shortened to fit a filename. + + ``Ledger`` stores an artifact as ``artifacts/.json``, so the canonical + string (a JSON document) can only be used as an id through a digest of it. + The full string is still what gets hashed -- no state is dropped. + """ + return f"lh-{content_sha(state)[:10]}" + + +# -- the check set ------------------------------------------------------------ + + +def check_tasks() -> Tuple[BenchmarkTask, ...]: + """The 18 checks the loop is measured on (and mostly gated on).""" + return tuple( + BenchmarkTask(task_id=f"{family}-{i}", family=family, + payload={"requires": list(requires)}) + for family, requires in FAMILIES.items() + for i in range(1, CHECKS_PER_FAMILY + 1) + ) + + +def fresh_tasks() -> Tuple[BenchmarkTask, ...]: + """Checks no step ever measured: the transfer set. + + They ask the same questions as the loop's checks under different ids, so + ``RSIBench.split`` would place them differently -- but they are used whole, + as a before/after pair, and never enter the split. + """ + return tuple( + BenchmarkTask(task_id=f"fresh-{family}-{i}", family=family, + payload={"requires": list(requires)}) + for family, requires in FAMILIES.items() + for i in range(1, FRESH_PER_FAMILY + 1) + ) + + +def check_score(task: BenchmarkTask, effective: Mapping[str, str]) -> float: + requires = task.payload["requires"] + matched = sum(1 for rule, body in requires if effective.get(rule) == body) + return matched / len(requires) + + +def score_map(tasks: Sequence[BenchmarkTask], + effective: Mapping[str, str]) -> Dict[str, float]: + return {task.task_id: check_score(task, effective) for task in tasks} + + +def counts(tasks: Sequence[BenchmarkTask], + effective: Mapping[str, str]) -> Tuple[float, float]: + """``(successes, failures)`` over the tasks, with partial checks counted. + + A chain check scores one half when its order is right but a dependency was + rewritten away, and a gate that only counts a check when it fully passes + cannot see chain progress at all: the first ``chain`` rule moves the check + from 0.0 to 0.5, never to 1.0, so the counts never move and the rule can + never be committed (measured: every chain step refused, every run). The + score *is* the measurement, so it is what the Beta posterior is fed; + ``BetaPosterior`` takes float counts and refuses a non-finite one. + """ + scores = [check_score(task, effective) for task in tasks] + successes = sum(scores) + return successes, len(scores) - successes + + +def reachable_ceiling(tasks: Sequence[BenchmarkTask]) -> float: + """The best mean score any assignment of rule bodies reaches on this set. + + Without it a retention of 0.44 looks like a broken loop; beside it, the + reader sees how much of the shortfall the conflicting families force. + """ + return max( + sum(check_score(task, assignment) for task in tasks) / len(tasks) + for assignment in ASSIGNMENTS + ) + + +# -- the proposer ------------------------------------------------------------- + + +def propose(family: str, effective: Mapping[str, str]) -> Tuple[str, str]: + """The one rule line that fixes this family -- and only this family. + + A scripted stand-in for a reflector: it reads the playbook and the family it + was pointed at, and returns the first requirement that is not in force. When + the family already passes it re-proposes the line already in force, which is + how a duplicate append reaches the gate instead of being filtered here. + """ + requires = FAMILIES[family] + for rule, body in requires: + if effective.get(rule) != body: + return rule, body + return requires[0] + + +def arrival_order(seed: int) -> Tuple[str, ...]: + """Which family arrives at which step. The **only** thing the seed changes.""" + order = list(FAMILIES) + random.Random(seed).shuffle(order) + return tuple(order) + + +# -- one seed ----------------------------------------------------------------- + + +def _serializer(artifact: Any) -> Dict[str, str]: + return dict(artifact.state) + + +def _deserializer(strategy: FileTree): + def load(aid: str, version: int, state: Dict[str, str]) -> EvolvingArtifact: + return EvolvingArtifact(aid, dict(state), version=version, strategy=strategy) + + return load + + +def _curve(task: BenchmarkTask, split: str, steps: Sequence[Mapping[str, float]], + errors: Sequence[Optional[str]]) -> TaskCurve: + return TaskCurve( + task_id=task.task_id, family=task.family, split=split, + scores=tuple(step[task.task_id] for step in steps), + oracle=tuple(None for _ in steps), + costs=tuple(0.0 for _ in steps), + errors=tuple(errors), + ) + + +def run_seed(seed: int, *, steps: int = 9, + holdout_fraction: float = HOLDOUT_FRACTION, + workdir: Optional[str] = None, + fail_step: Optional[int] = None) -> Dict[str, Any]: + """One continual run: one artifact, ``steps`` sequential rewrites.""" + started = time.perf_counter() + tasks = check_tasks() + # The RSIBench instance is used for its hash split only; this loop's clock + # is the step, not the generation, so `generations` is just the floor it + # validates. + bench = RSIBench(tasks, generations=max(2, steps), + holdout_fraction=holdout_fraction) + train, holdout = bench.split() + split_of = {task.task_id: "train" for task in train} + split_of.update({task.task_id: "holdout" for task in holdout}) + holdout_set = tuple(holdout) + + tree = FileTree(initial_files={PLAYBOOK: HEADER}, max_files_per_diff=1) + state = tree.initial() + history = [score_map(tasks, parse_playbook(state[PLAYBOOK]))] + errors: List[Optional[str]] = [None] + evaluations = 0 + commits = 0 + order = arrival_order(seed) + gate = DefaultAcceptance(**GATE) + records: List[Dict[str, Any]] = [] + + repo = tempfile.mkdtemp(prefix=f"long-horizon-{seed}-", dir=workdir) + ledger = Ledger(repo, _serializer, _deserializer(tree)) + artifact = EvolvingArtifact(artifact_id(state), dict(state), strategy=tree) + ledger.register(artifact) + commits += 1 # the registration is the genesis node of the lineage + + try: + for step_no in range(1, steps + 1): + phase = order[(step_no - 1) % len(order)] + effective = parse_playbook(state[PLAYBOOK]) + rule, body = propose(phase, effective) + new_text = append_rule(state[PLAYBOOK], rule, body) + candidate_state = dict(state) + candidate_state[PLAYBOOK] = new_text + proposal = json.dumps({ + "rationale": f"{phase} fails; set {rule}: {body}", + "edits": [{"path": PLAYBOOK, "content": new_text}], + }) + diff = tree.to_diff(state, proposal, "scripted-proposer", + artifact.version, artifact.id) + record: Dict[str, Any] = { + "step": step_no, "phase": phase, "rule": rule, "body": body, + "ok": True, "error": None, "committed": False, + "parent_sha": content_sha(state), "child_sha": content_sha(state), + "version_before": artifact.version, "version_after": artifact.version, + "holdout_before": 0.0, "holdout_after": 0.0, + "p_improve": 0.0, "gate": "", "reason": "", + } + try: + if fail_step == step_no: + raise RuntimeError(f"injected failure at step {step_no}") + base_counts = counts(holdout_set, effective) + evaluations += len(holdout_set) + candidate_effective = parse_playbook(new_text) + cand_counts = counts(holdout_set, candidate_effective) + evaluations += len(holdout_set) + candidate = EvolvingArtifact(artifact.id, candidate_state, + version=artifact.version + 1, + strategy=tree) + before = MergeContext.rate(base_counts) + after = MergeContext.rate(cand_counts) + decision = gate.accept(MergeContext( + artifact=artifact, candidate=candidate, cards=(), + base_counts=base_counts, cand_counts=cand_counts, diff=diff)) + record.update({ + "holdout_before": before, "holdout_after": after, + "p_improve": decision.p_improve, + "gate": decision.category, + "reason": decision.detail or decision.category, + }) + if decision.accept: + head = ledger.head_version() + _, version = ledger.commit( + candidate, head, + message=(f"step {step_no}: {phase} -> {rule}: {body} " + f"(held-out {before:.2f} -> {after:.2f})")) + commits += 1 + artifact = EvolvingArtifact(artifact.id, candidate_state, + version=version, strategy=tree) + state = candidate_state + record.update({ + "committed": True, "child_sha": content_sha(state), + "version_after": version, + }) + except Exception as exc: # noqa: BLE001 - a failed step is a data point + record.update({"ok": False, "gate": "raised", + "error": f"{type(exc).__name__}: {exc}", + "reason": "step raised; state left unchanged"}) + effective = parse_playbook(state[PLAYBOOK]) + step_scores = score_map(tasks, effective) + evaluations += len(tasks) + history.append(step_scores) + errors.append(record["error"]) + records.append(record) + finally: + ledger.close() + + curves = tuple(_curve(task, split_of[task.task_id], history, errors) + for task in sorted(tasks, key=lambda item: item.task_id)) + train_curves = [c for c in curves if c.split == "train"] + holdout_curves = [c for c in curves if c.split == "holdout"] + train_gain = _mean([c.gain for c in train_curves]) + holdout_gain = _mean([c.gain for c in holdout_curves]) + forgetting = _mean([c.forgetting for c in curves]) + max_forgetting = max((c.forgetting for c in curves), default=0.0) + forgetting_fraction = ( + sum(1 for c in curves if c.forgetting > 0.05) / len(curves) if curves else 0.0) + error_count = sum(c.error_count for c in curves) + final = parse_playbook(state[PLAYBOOK]) + final_scores = score_map(tasks, final) + retention = _mean(list(final_scores.values())) + + fresh = fresh_tasks() + before = score_map(fresh, parse_playbook(tree.initial()[PLAYBOOK])) + after = score_map(fresh, final) + evaluations += 2 * len(fresh) + ci = bootstrap_ci(before, after, n=2000, seed=seed) + + if error_count: + verdict = "errored" + elif max_forgetting > 0.05: + verdict = "forgetting" + elif train_gain > 0.05 and ci.mean <= 1e-9: + verdict = "no-transfer" + elif ci.mean > 1e-9: + verdict = "improving" + elif ci.mean < -1e-9: + verdict = "regressing" + else: + verdict = "flat" + + committed_now = sum(1 for record in records if record["committed"]) + report = RSIBenchReport( + curves=curves, generations=len(history), + train_gain=train_gain, holdout_gain=holdout_gain, transfer=ci.mean, + forgetting=forgetting, max_forgetting=max_forgetting, + forgetting_fraction=forgetting_fraction, reward_hacking_gap=None, + cost=float(evaluations), generations_to_target=None, errors=error_count, + verdict=verdict, + detail=(f"{steps} step(s) over {len(tasks)} check(s), " + f"{len(holdout_set)} held out, {committed_now} rewrite(s) " + f"committed; retention={retention:.3f}, " + f"max-forget={max_forgetting:.3f}, transfer={ci.mean:+.3f}"), + ) + families_in_holdout = sorted({task.family for task in holdout_set}) + return { + "seed": seed, + "ok": error_count == 0, + "verdict": verdict, + "retention": retention, + "ceiling": reachable_ceiling(tasks), + "forgetting": forgetting, + "max_forgetting": max_forgetting, + "forgetting_fraction": forgetting_fraction, + "committed": committed_now, + "refused": sum(1 for record in records + if record["ok"] and not record["committed"]), + "failed": sum(1 for record in records if not record["ok"]), + "arrival_order": list(order), + "transfer": { + "mean": ci.mean, "lo": ci.lo, "hi": ci.hi, + "prob_positive": ci.prob_positive, "pairs": ci.n, + "definition": "paired final minus step-0 on the fresh check set, " + "bootstrapped over checks within this seed", + }, + "families_without_holdout": [ + family for family in FAMILIES if family not in families_in_holdout], + "cost": { + "check_evaluations": evaluations, + "ledger_commits": commits, + "artifact_bytes": len(state[PLAYBOOK].encode("utf-8")), + "artifact_lines": len( + [line for line in state[PLAYBOOK].splitlines() + if line.startswith("- ")]), + "wall_clock_s": round(time.perf_counter() - started, 3), + }, + "final_rules": dict(sorted(final.items())), + "steps": records, + "lineage": [record for record in records if record["committed"]], + "report": report.to_dict(), + } + + +def _mean(values: Sequence[float]) -> float: + return sum(values) / len(values) if values else 0.0 + + +# -- the whole run ------------------------------------------------------------ + + +def run_bench(*, seeds: int = 3, steps: int = 9, + holdout_fraction: float = HOLDOUT_FRACTION, + workdir: Optional[str] = None, + fail_step: Optional[int] = None, + code_version: str = "") -> Dict[str, Any]: + """Every seed, the manifest they share, and the spread across them.""" + if isinstance(steps, bool) or not isinstance(steps, int) \ + or not 1 <= steps <= MAX_STEPS: + raise ValueError(f"steps must be an int in 1..{MAX_STEPS}") + if isinstance(seeds, bool) or not isinstance(seeds, int) or seeds < 1: + raise ValueError("seeds must be a positive int") + + seed_numbers = tuple(range(seeds)) + manifest = build_manifest( + { + "algorithm": {"name": "long-horizon-continual"}, + "evolve": {"seeds": list(seed_numbers)}, + "data": {"path": ""}, + }, + data_paths=[], + model_config={"model": "none (offline scripted proposer)"}, + code_version=code_version, + extra={ + "bench": BENCH, "steps": steps, + "holdout_fraction": holdout_fraction, + "proposer": "scripted-offline", "gate": "DefaultAcceptance", + }, + ) + runs = [run_seed(seed, steps=steps, holdout_fraction=holdout_fraction, + workdir=workdir, fail_step=fail_step) + for seed in seed_numbers] + + tasks = check_tasks() + train, holdout = RSIBench(tasks, holdout_fraction=holdout_fraction).split() + verdicts: Dict[str, int] = {} + for run in runs: + verdicts[run["verdict"]] = verdicts.get(run["verdict"], 0) + 1 + # Verdict order, weakest evidence first: a single seed that errored or lost a + # family to forgetting outranks a seed that improved (#1177 review). + for verdict in ("errored", "forgetting", "regressing", "no-transfer", + "flat", "improving"): + if verdict in verdicts: + overall = verdict + break + else: + overall = "insufficient" + + return { + "bench": BENCH, + "version": VERSION, + "manifest_id": manifest.manifest_id, + "spec_hash": manifest.spec_hash, + "code_version": manifest.code_version, + "model": dict(manifest.model_config), + "steps": steps, + "seeds": list(seed_numbers), + "checks": { + "total": len(tasks), + "train": len(train), + "holdout": len(holdout), + "holdout_ids": sorted(task.task_id for task in holdout), + "holdout_by_family": { + family: sum(1 for task in holdout if task.family == family) + for family in FAMILIES + }, + "families_without_holdout": [ + family for family in FAMILIES + if not any(task.family == family for task in holdout)], + "ceiling": reachable_ceiling(tasks), + }, + "runs": runs, + "summary": { + "retention": {"min": min(r["retention"] for r in runs), + "max": max(r["retention"] for r in runs)}, + "forgetting": {"min": min(r["forgetting"] for r in runs), + "max": max(r["forgetting"] for r in runs)}, + "max_forgetting": max(r["max_forgetting"] for r in runs), + "transfer": { + "min": min(r["transfer"]["mean"] for r in runs), + "max": max(r["transfer"]["mean"] for r in runs), + }, + "committed": sum(r["committed"] for r in runs), + "refused": sum(r["refused"] for r in runs), + "failed": sum(r["failed"] for r in runs), + "verdicts": verdicts, + }, + "verdict": overall, + "all_ok": all(run["ok"] for run in runs), + "notes": [ + "three seeds are a range, not significance: each seed shuffles which " + "family arrives when, and the report gives the min and max across them", + f"the check families conflict by construction; the best assignment of " + f"rule bodies reaches {reachable_ceiling(tasks):.3f} retention, so a " + f"shortfall below that is the loop's, not the domain's", + "the gate reads the held-out checks only; a family with no held-out " + "check is not protected, which is what families_without_holdout names", + "no proxy/oracle pair exists in this domain, so the verdict never says " + "hacking and reward_hacking_gap is null", + ], + } + + +def _print_human(report: Mapping[str, Any]) -> None: + checks = report["checks"] + print(f"[{BENCH}] {report['steps']} step(s), {len(report['seeds'])} seed(s), " + f"{checks['total']} check(s) ({checks['holdout']} held out), " + f"ceiling {checks['ceiling']:.3f}") + print(f"[{BENCH}] manifest {report['manifest_id'][:16]} " + f"model {report['model'].get('model', '?')}") + for run in report["runs"]: + status = "ok " if run["ok"] else "FAIL" + transfer = run["transfer"] + print( + f" [{status}] seed {run['seed']}: retention {run['retention']:.3f}, " + f"forgetting {run['forgetting']:.3f} (max {run['max_forgetting']:.3f}), " + f"transfer {transfer['mean']:+.3f} " + f"[{transfer['lo']:+.3f}, {transfer['hi']:+.3f}], " + f"{run['committed']} committed / {run['refused']} refused / " + f"{run['failed']} failed, verdict {run['verdict']}") + for record in run["steps"]: + mark = "commit " if record["committed"] else ( + "FAIL " if not record["ok"] else "refuse ") + print(f" step {record['step']:>2} {record['phase']:<12} " + f"{mark} {record['rule']}: {record['body']} {record['reason']}") + print(f"[{BENCH}] verdict: {report['verdict']}") + + +def main(argv: Optional[Sequence[str]] = None) -> int: + from agentdescent import __version__ + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--steps", type=int, default=9, + help=f"sequential rewrites of the one artifact " + f"(1..{MAX_STEPS}, default 9; published runs use " + f"5-15)") + parser.add_argument("--seeds", type=int, default=3, + help="how many arrival orders to run (default 3)") + parser.add_argument("--holdout-fraction", type=float, + default=HOLDOUT_FRACTION, + help="share of the checks the gate sees " + f"(default {HOLDOUT_FRACTION})") + parser.add_argument("--json", action="store_true", + help="print only the machine-readable report") + parser.add_argument("--out", default="", + help="also write the report to this path") + parser.add_argument("--fail-step", type=int, default=None, + help="raise inside this step, to show a failed step is " + "recorded rather than crashing the run") + parser.add_argument("--workdir", default=None, + help="parent for the temporary git ledgers") + args = parser.parse_args(argv) + if not 1 <= args.steps <= MAX_STEPS: + parser.error(f"--steps must be in 1..{MAX_STEPS}") + if args.seeds < 1: + parser.error("--seeds must be >= 1") + + report = run_bench(seeds=args.seeds, steps=args.steps, + holdout_fraction=args.holdout_fraction, + workdir=args.workdir, fail_step=args.fail_step, + code_version=__version__) + text = json.dumps(report, indent=2, ensure_ascii=False, default=str) + if args.out: + with open(args.out, "w", encoding="utf-8") as handle: + handle.write(text + "\n") + if args.json: + print(text) + else: + _print_human(report) + return 0 if report["all_ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/results/long-horizon-3seed.json b/bench/results/long-horizon-3seed.json new file mode 100644 index 00000000..2c48280f --- /dev/null +++ b/bench/results/long-horizon-3seed.json @@ -0,0 +1,3967 @@ +{ + "bench": "long_horizon", + "version": 1, + "manifest_id": "455feebf7275f37d1f40059fc3e339b7f42fbdec64657840509077e048d07c44", + "spec_hash": "b3139220ada8048480a9beb4aed88786b01e724de96dec9df849d32018f4b73c", + "code_version": "0.11.0", + "model": { + "model": "none (offline scripted proposer)" + }, + "steps": 9, + "seeds": [ + 0, + 1, + 2 + ], + "checks": { + "total": 18, + "train": 12, + "holdout": 6, + "holdout_ids": [ + "add_product-2", + "add_sum-2", + "chain_add-2", + "chain_mul-2", + "mul_product-1", + "mul_sum-1" + ], + "holdout_by_family": { + "add_sum": 1, + "add_product": 1, + "mul_sum": 1, + "mul_product": 1, + "chain_add": 1, + "chain_mul": 1 + }, + "families_without_holdout": [], + "ceiling": 0.5833333333333334 + }, + "runs": [ + { + "seed": 0, + "ok": true, + "verdict": "forgetting", + "retention": 0.5833333333333334, + "ceiling": 0.5833333333333334, + "forgetting": 0.3333333333333333, + "max_forgetting": 1.0, + "forgetting_fraction": 0.3333333333333333, + "committed": 5, + "refused": 4, + "failed": 0, + "arrival_order": [ + "chain_add", + "mul_sum", + "add_product", + "add_sum", + "chain_mul", + "mul_product" + ], + "transfer": { + "mean": 0.5833333333333334, + "lo": 0.3333333333333333, + "hi": 0.8333333333333334, + "prob_positive": 1.0, + "pairs": 12, + "definition": "paired final minus step-0 on the fresh check set, bootstrapped over checks within this seed" + }, + "families_without_holdout": [], + "cost": { + "check_evaluations": 294, + "ledger_commits": 6, + "artifact_bytes": 210, + "artifact_lines": 5, + "wall_clock_s": 0.556 + }, + "final_rules": { + "add": "sum", + "chain": "add_then_mul", + "mul": "product" + }, + "steps": [ + { + "step": 1, + "phase": "chain_add", + "rule": "chain", + "body": "add_then_mul", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "f8fc2929d933", + "child_sha": "5efb5ba9ae58", + "version_before": 1, + "version_after": 2, + "holdout_before": 0.0, + "holdout_after": 0.08333333333333333, + "p_improve": 0.652, + "gate": "committed", + "reason": "committed" + }, + { + "step": 2, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "5efb5ba9ae58", + "child_sha": "f248f4a7aef7", + "version_before": 2, + "version_after": 3, + "holdout_before": 0.08333333333333333, + "holdout_after": 0.25, + "p_improve": 0.73925, + "gate": "committed", + "reason": "committed" + }, + { + "step": 3, + "phase": "add_product", + "rule": "add", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "f248f4a7aef7", + "child_sha": "4444d9dafe40", + "version_before": 3, + "version_after": 4, + "holdout_before": 0.25, + "holdout_after": 0.4166666666666667, + "p_improve": 0.707, + "gate": "committed", + "reason": "committed" + }, + { + "step": 4, + "phase": "add_sum", + "rule": "add", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "4444d9dafe40", + "child_sha": "b0b76126fad9", + "version_before": 4, + "version_after": 5, + "holdout_before": 0.4166666666666667, + "holdout_after": 0.5, + "p_improve": 0.606, + "gate": "committed", + "reason": "committed" + }, + { + "step": 5, + "phase": "chain_mul", + "rule": "chain", + "body": "mul_then_add", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "b0b76126fad9", + "child_sha": "b0b76126fad9", + "version_before": 5, + "version_after": 5, + "holdout_before": 0.5, + "holdout_after": 0.5, + "p_improve": 0.5105, + "gate": "below-threshold", + "reason": "P(delta>0)=0.51 <= 0.53" + }, + { + "step": 6, + "phase": "mul_product", + "rule": "mul", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "b0b76126fad9", + "child_sha": "c38315f0d31f", + "version_before": 5, + "version_after": 6, + "holdout_before": 0.5, + "holdout_after": 0.5833333333333334, + "p_improve": 0.6, + "gate": "committed", + "reason": "committed" + }, + { + "step": 7, + "phase": "chain_add", + "rule": "chain", + "body": "add_then_mul", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "c38315f0d31f", + "child_sha": "c38315f0d31f", + "version_before": 6, + "version_after": 6, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5833333333333334, + "p_improve": 0.4925, + "gate": "below-threshold", + "reason": "P(delta>0)=0.49 <= 0.53" + }, + { + "step": 8, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "c38315f0d31f", + "child_sha": "c38315f0d31f", + "version_before": 6, + "version_after": 6, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5, + "p_improve": 0.38475, + "gate": "below-threshold", + "reason": "held-out regression 0.583 -> 0.500" + }, + { + "step": 9, + "phase": "add_product", + "rule": "add", + "body": "product", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "c38315f0d31f", + "child_sha": "c38315f0d31f", + "version_before": 6, + "version_after": 6, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5, + "p_improve": 0.3925, + "gate": "below-threshold", + "reason": "held-out regression 0.583 -> 0.500" + } + ], + "lineage": [ + { + "step": 1, + "phase": "chain_add", + "rule": "chain", + "body": "add_then_mul", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "f8fc2929d933", + "child_sha": "5efb5ba9ae58", + "version_before": 1, + "version_after": 2, + "holdout_before": 0.0, + "holdout_after": 0.08333333333333333, + "p_improve": 0.652, + "gate": "committed", + "reason": "committed" + }, + { + "step": 2, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "5efb5ba9ae58", + "child_sha": "f248f4a7aef7", + "version_before": 2, + "version_after": 3, + "holdout_before": 0.08333333333333333, + "holdout_after": 0.25, + "p_improve": 0.73925, + "gate": "committed", + "reason": "committed" + }, + { + "step": 3, + "phase": "add_product", + "rule": "add", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "f248f4a7aef7", + "child_sha": "4444d9dafe40", + "version_before": 3, + "version_after": 4, + "holdout_before": 0.25, + "holdout_after": 0.4166666666666667, + "p_improve": 0.707, + "gate": "committed", + "reason": "committed" + }, + { + "step": 4, + "phase": "add_sum", + "rule": "add", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "4444d9dafe40", + "child_sha": "b0b76126fad9", + "version_before": 4, + "version_after": 5, + "holdout_before": 0.4166666666666667, + "holdout_after": 0.5, + "p_improve": 0.606, + "gate": "committed", + "reason": "committed" + }, + { + "step": 6, + "phase": "mul_product", + "rule": "mul", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "b0b76126fad9", + "child_sha": "c38315f0d31f", + "version_before": 5, + "version_after": 6, + "holdout_before": 0.5, + "holdout_after": 0.5833333333333334, + "p_improve": 0.6, + "gate": "committed", + "reason": "committed" + } + ], + "report": { + "generations": 10, + "train_gain": 0.5833333333333334, + "holdout_gain": 0.5833333333333334, + "transfer": 0.5833333333333334, + "forgetting": 0.3333333333333333, + "max_forgetting": 1.0, + "forgetting_fraction": 0.3333333333333333, + "reward_hacking_gap": null, + "cost": 294.0, + "generations_to_target": null, + "errors": 0, + "verdict": "forgetting", + "detail": "9 step(s) over 18 check(s), 6 held out, 5 rewrite(s) committed; retention=0.583, max-forget=1.000, transfer=+0.583", + "tasks": [ + { + "task_id": "add_product-1", + "family": "add_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "add_product-2", + "family": "add_product", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "add_product-3", + "family": "add_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "add_sum-1", + "family": "add_sum", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "add_sum-2", + "family": "add_sum", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "add_sum-3", + "family": "add_sum", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-1", + "family": "chain_add", + "split": "train", + "scores": [ + 0.0, + 0.5, + 0.5, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-2", + "family": "chain_add", + "split": "holdout", + "scores": [ + 0.0, + 0.5, + 0.5, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-3", + "family": "chain_add", + "split": "train", + "scores": [ + 0.0, + 0.5, + 0.5, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-1", + "family": "chain_mul", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-2", + "family": "chain_mul", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-3", + "family": "chain_mul", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-1", + "family": "mul_product", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-2", + "family": "mul_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-3", + "family": "mul_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_sum-1", + "family": "mul_sum", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "mul_sum-2", + "family": "mul_sum", + "split": "train", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "mul_sum-3", + "family": "mul_sum", + "split": "train", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + } + ] + } + }, + { + "seed": 1, + "ok": true, + "verdict": "forgetting", + "retention": 0.5833333333333334, + "ceiling": 0.5833333333333334, + "forgetting": 0.16666666666666666, + "max_forgetting": 1.0, + "forgetting_fraction": 0.16666666666666666, + "committed": 4, + "refused": 5, + "failed": 0, + "arrival_order": [ + "mul_sum", + "mul_product", + "chain_mul", + "add_sum", + "chain_add", + "add_product" + ], + "transfer": { + "mean": 0.5833333333333334, + "lo": 0.3333333333333333, + "hi": 0.8333333333333334, + "prob_positive": 1.0, + "pairs": 12, + "definition": "paired final minus step-0 on the fresh check set, bootstrapped over checks within this seed" + }, + "families_without_holdout": [], + "cost": { + "check_evaluations": 294, + "ledger_commits": 5, + "artifact_bytes": 195, + "artifact_lines": 4, + "wall_clock_s": 0.501 + }, + "final_rules": { + "add": "sum", + "chain": "mul_then_add", + "mul": "product" + }, + "steps": [ + { + "step": 1, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "f8fc2929d933", + "child_sha": "8d1dc76dc912", + "version_before": 1, + "version_after": 2, + "holdout_before": 0.0, + "holdout_after": 0.16666666666666666, + "p_improve": 0.775, + "gate": "committed", + "reason": "committed" + }, + { + "step": 2, + "phase": "mul_product", + "rule": "mul", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "8d1dc76dc912", + "child_sha": "d90a9db2fcbd", + "version_before": 2, + "version_after": 3, + "holdout_before": 0.16666666666666666, + "holdout_after": 0.25, + "p_improve": 0.633, + "gate": "committed", + "reason": "committed" + }, + { + "step": 3, + "phase": "chain_mul", + "rule": "chain", + "body": "mul_then_add", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "d90a9db2fcbd", + "child_sha": "2c163b87e1ff", + "version_before": 3, + "version_after": 4, + "holdout_before": 0.25, + "holdout_after": 0.3333333333333333, + "p_improve": 0.61675, + "gate": "committed", + "reason": "committed" + }, + { + "step": 4, + "phase": "add_sum", + "rule": "add", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "2c163b87e1ff", + "child_sha": "e7b1948324c7", + "version_before": 4, + "version_after": 5, + "holdout_before": 0.3333333333333333, + "holdout_after": 0.5833333333333334, + "p_improve": 0.79275, + "gate": "committed", + "reason": "committed" + }, + { + "step": 5, + "phase": "chain_add", + "rule": "chain", + "body": "add_then_mul", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "e7b1948324c7", + "child_sha": "e7b1948324c7", + "version_before": 5, + "version_after": 5, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5833333333333334, + "p_improve": 0.5035, + "gate": "below-threshold", + "reason": "P(delta>0)=0.50 <= 0.53" + }, + { + "step": 6, + "phase": "add_product", + "rule": "add", + "body": "product", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "e7b1948324c7", + "child_sha": "e7b1948324c7", + "version_before": 5, + "version_after": 5, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5, + "p_improve": 0.4055, + "gate": "below-threshold", + "reason": "held-out regression 0.583 -> 0.500" + }, + { + "step": 7, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "e7b1948324c7", + "child_sha": "e7b1948324c7", + "version_before": 5, + "version_after": 5, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5, + "p_improve": 0.39, + "gate": "below-threshold", + "reason": "held-out regression 0.583 -> 0.500" + }, + { + "step": 8, + "phase": "mul_product", + "rule": "mul", + "body": "product", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "e7b1948324c7", + "child_sha": "e7b1948324c7", + "version_before": 5, + "version_after": 5, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5833333333333334, + "p_improve": 0.50625, + "gate": "below-threshold", + "reason": "P(delta>0)=0.51 <= 0.53" + }, + { + "step": 9, + "phase": "chain_mul", + "rule": "chain", + "body": "mul_then_add", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "e7b1948324c7", + "child_sha": "e7b1948324c7", + "version_before": 5, + "version_after": 5, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5833333333333334, + "p_improve": 0.4985, + "gate": "below-threshold", + "reason": "P(delta>0)=0.50 <= 0.53" + } + ], + "lineage": [ + { + "step": 1, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "f8fc2929d933", + "child_sha": "8d1dc76dc912", + "version_before": 1, + "version_after": 2, + "holdout_before": 0.0, + "holdout_after": 0.16666666666666666, + "p_improve": 0.775, + "gate": "committed", + "reason": "committed" + }, + { + "step": 2, + "phase": "mul_product", + "rule": "mul", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "8d1dc76dc912", + "child_sha": "d90a9db2fcbd", + "version_before": 2, + "version_after": 3, + "holdout_before": 0.16666666666666666, + "holdout_after": 0.25, + "p_improve": 0.633, + "gate": "committed", + "reason": "committed" + }, + { + "step": 3, + "phase": "chain_mul", + "rule": "chain", + "body": "mul_then_add", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "d90a9db2fcbd", + "child_sha": "2c163b87e1ff", + "version_before": 3, + "version_after": 4, + "holdout_before": 0.25, + "holdout_after": 0.3333333333333333, + "p_improve": 0.61675, + "gate": "committed", + "reason": "committed" + }, + { + "step": 4, + "phase": "add_sum", + "rule": "add", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "2c163b87e1ff", + "child_sha": "e7b1948324c7", + "version_before": 4, + "version_after": 5, + "holdout_before": 0.3333333333333333, + "holdout_after": 0.5833333333333334, + "p_improve": 0.79275, + "gate": "committed", + "reason": "committed" + } + ], + "report": { + "generations": 10, + "train_gain": 0.5833333333333334, + "holdout_gain": 0.5833333333333334, + "transfer": 0.5833333333333334, + "forgetting": 0.16666666666666666, + "max_forgetting": 1.0, + "forgetting_fraction": 0.16666666666666666, + "reward_hacking_gap": null, + "cost": 294.0, + "generations_to_target": null, + "errors": 0, + "verdict": "forgetting", + "detail": "9 step(s) over 18 check(s), 6 held out, 4 rewrite(s) committed; retention=0.583, max-forget=1.000, transfer=+0.583", + "tasks": [ + { + "task_id": "add_product-1", + "family": "add_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 0.0 + }, + { + "task_id": "add_product-2", + "family": "add_product", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 0.0 + }, + { + "task_id": "add_product-3", + "family": "add_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 0.0 + }, + { + "task_id": "add_sum-1", + "family": "add_sum", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "add_sum-2", + "family": "add_sum", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "add_sum-3", + "family": "add_sum", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-1", + "family": "chain_add", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-2", + "family": "chain_add", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-3", + "family": "chain_add", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-1", + "family": "chain_mul", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-2", + "family": "chain_mul", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-3", + "family": "chain_mul", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-1", + "family": "mul_product", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-2", + "family": "mul_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-3", + "family": "mul_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_sum-1", + "family": "mul_sum", + "split": "holdout", + "scores": [ + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "mul_sum-2", + "family": "mul_sum", + "split": "train", + "scores": [ + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "mul_sum-3", + "family": "mul_sum", + "split": "train", + "scores": [ + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + } + ] + } + }, + { + "seed": 2, + "ok": true, + "verdict": "forgetting", + "retention": 0.5833333333333334, + "ceiling": 0.5833333333333334, + "forgetting": 0.3333333333333333, + "max_forgetting": 1.0, + "forgetting_fraction": 0.3333333333333333, + "committed": 5, + "refused": 4, + "failed": 0, + "arrival_order": [ + "mul_sum", + "mul_product", + "add_product", + "chain_add", + "chain_mul", + "add_sum" + ], + "transfer": { + "mean": 0.5833333333333334, + "lo": 0.3333333333333333, + "hi": 0.8333333333333334, + "prob_positive": 1.0, + "pairs": 12, + "definition": "paired final minus step-0 on the fresh check set, bootstrapped over checks within this seed" + }, + "families_without_holdout": [], + "cost": { + "check_evaluations": 294, + "ledger_commits": 6, + "artifact_bytes": 210, + "artifact_lines": 5, + "wall_clock_s": 0.595 + }, + "final_rules": { + "add": "sum", + "chain": "add_then_mul", + "mul": "product" + }, + "steps": [ + { + "step": 1, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "f8fc2929d933", + "child_sha": "8d1dc76dc912", + "version_before": 1, + "version_after": 2, + "holdout_before": 0.0, + "holdout_after": 0.16666666666666666, + "p_improve": 0.775, + "gate": "committed", + "reason": "committed" + }, + { + "step": 2, + "phase": "mul_product", + "rule": "mul", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "8d1dc76dc912", + "child_sha": "d90a9db2fcbd", + "version_before": 2, + "version_after": 3, + "holdout_before": 0.16666666666666666, + "holdout_after": 0.25, + "p_improve": 0.633, + "gate": "committed", + "reason": "committed" + }, + { + "step": 3, + "phase": "add_product", + "rule": "add", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "d90a9db2fcbd", + "child_sha": "a40352ce6df4", + "version_before": 3, + "version_after": 4, + "holdout_before": 0.25, + "holdout_after": 0.4166666666666667, + "p_improve": 0.70675, + "gate": "committed", + "reason": "committed" + }, + { + "step": 4, + "phase": "chain_add", + "rule": "chain", + "body": "add_then_mul", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "a40352ce6df4", + "child_sha": "1953f7ada60a", + "version_before": 4, + "version_after": 5, + "holdout_before": 0.4166666666666667, + "holdout_after": 0.5, + "p_improve": 0.60525, + "gate": "committed", + "reason": "committed" + }, + { + "step": 5, + "phase": "chain_mul", + "rule": "chain", + "body": "mul_then_add", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "1953f7ada60a", + "child_sha": "1953f7ada60a", + "version_before": 5, + "version_after": 5, + "holdout_before": 0.5, + "holdout_after": 0.5, + "p_improve": 0.4975, + "gate": "below-threshold", + "reason": "P(delta>0)=0.50 <= 0.53" + }, + { + "step": 6, + "phase": "add_sum", + "rule": "add", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "1953f7ada60a", + "child_sha": "23d13a436d25", + "version_before": 5, + "version_after": 6, + "holdout_before": 0.5, + "holdout_after": 0.5833333333333334, + "p_improve": 0.60575, + "gate": "committed", + "reason": "committed" + }, + { + "step": 7, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "23d13a436d25", + "child_sha": "23d13a436d25", + "version_before": 6, + "version_after": 6, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5, + "p_improve": 0.38475, + "gate": "below-threshold", + "reason": "held-out regression 0.583 -> 0.500" + }, + { + "step": 8, + "phase": "mul_product", + "rule": "mul", + "body": "product", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "23d13a436d25", + "child_sha": "23d13a436d25", + "version_before": 6, + "version_after": 6, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5833333333333334, + "p_improve": 0.50025, + "gate": "below-threshold", + "reason": "P(delta>0)=0.50 <= 0.53" + }, + { + "step": 9, + "phase": "add_product", + "rule": "add", + "body": "product", + "ok": true, + "error": null, + "committed": false, + "parent_sha": "23d13a436d25", + "child_sha": "23d13a436d25", + "version_before": 6, + "version_after": 6, + "holdout_before": 0.5833333333333334, + "holdout_after": 0.5, + "p_improve": 0.3925, + "gate": "below-threshold", + "reason": "held-out regression 0.583 -> 0.500" + } + ], + "lineage": [ + { + "step": 1, + "phase": "mul_sum", + "rule": "mul", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "f8fc2929d933", + "child_sha": "8d1dc76dc912", + "version_before": 1, + "version_after": 2, + "holdout_before": 0.0, + "holdout_after": 0.16666666666666666, + "p_improve": 0.775, + "gate": "committed", + "reason": "committed" + }, + { + "step": 2, + "phase": "mul_product", + "rule": "mul", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "8d1dc76dc912", + "child_sha": "d90a9db2fcbd", + "version_before": 2, + "version_after": 3, + "holdout_before": 0.16666666666666666, + "holdout_after": 0.25, + "p_improve": 0.633, + "gate": "committed", + "reason": "committed" + }, + { + "step": 3, + "phase": "add_product", + "rule": "add", + "body": "product", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "d90a9db2fcbd", + "child_sha": "a40352ce6df4", + "version_before": 3, + "version_after": 4, + "holdout_before": 0.25, + "holdout_after": 0.4166666666666667, + "p_improve": 0.70675, + "gate": "committed", + "reason": "committed" + }, + { + "step": 4, + "phase": "chain_add", + "rule": "chain", + "body": "add_then_mul", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "a40352ce6df4", + "child_sha": "1953f7ada60a", + "version_before": 4, + "version_after": 5, + "holdout_before": 0.4166666666666667, + "holdout_after": 0.5, + "p_improve": 0.60525, + "gate": "committed", + "reason": "committed" + }, + { + "step": 6, + "phase": "add_sum", + "rule": "add", + "body": "sum", + "ok": true, + "error": null, + "committed": true, + "parent_sha": "1953f7ada60a", + "child_sha": "23d13a436d25", + "version_before": 5, + "version_after": 6, + "holdout_before": 0.5, + "holdout_after": 0.5833333333333334, + "p_improve": 0.60575, + "gate": "committed", + "reason": "committed" + } + ], + "report": { + "generations": 10, + "train_gain": 0.5833333333333334, + "holdout_gain": 0.5833333333333334, + "transfer": 0.5833333333333334, + "forgetting": 0.3333333333333333, + "max_forgetting": 1.0, + "forgetting_fraction": 0.3333333333333333, + "reward_hacking_gap": null, + "cost": 294.0, + "generations_to_target": null, + "errors": 0, + "verdict": "forgetting", + "detail": "9 step(s) over 18 check(s), 6 held out, 5 rewrite(s) committed; retention=0.583, max-forget=1.000, transfer=+0.583", + "tasks": [ + { + "task_id": "add_product-1", + "family": "add_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "add_product-2", + "family": "add_product", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "add_product-3", + "family": "add_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "add_sum-1", + "family": "add_sum", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "add_sum-2", + "family": "add_sum", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "add_sum-3", + "family": "add_sum", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-1", + "family": "chain_add", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-2", + "family": "chain_add", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_add-3", + "family": "chain_add", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.5, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-1", + "family": "chain_mul", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-2", + "family": "chain_mul", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "chain_mul-3", + "family": "chain_mul", + "split": "train", + "scores": [ + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.5, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-1", + "family": "mul_product", + "split": "holdout", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-2", + "family": "mul_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_product-3", + "family": "mul_product", + "split": "train", + "scores": [ + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 1.0, + "forgetting": 0.0 + }, + { + "task_id": "mul_sum-1", + "family": "mul_sum", + "split": "holdout", + "scores": [ + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "mul_sum-2", + "family": "mul_sum", + "split": "train", + "scores": [ + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + }, + { + "task_id": "mul_sum-3", + "family": "mul_sum", + "split": "train", + "scores": [ + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "oracle": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "costs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "errors": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "gain": 0.0, + "forgetting": 1.0 + } + ] + } + } + ], + "summary": { + "retention": { + "min": 0.5833333333333334, + "max": 0.5833333333333334 + }, + "forgetting": { + "min": 0.16666666666666666, + "max": 0.3333333333333333 + }, + "max_forgetting": 1.0, + "transfer": { + "min": 0.5833333333333334, + "max": 0.5833333333333334 + }, + "committed": 14, + "refused": 13, + "failed": 0, + "verdicts": { + "forgetting": 3 + } + }, + "verdict": "forgetting", + "all_ok": true, + "notes": [ + "three seeds are a range, not significance: each seed shuffles which family arrives when, and the report gives the min and max across them", + "the check families conflict by construction; the best assignment of rule bodies reaches 0.583 retention, so a shortfall below that is the loop's, not the domain's", + "the gate reads the held-out checks only; a family with no held-out check is not protected, which is what families_without_holdout names", + "no proxy/oracle pair exists in this domain, so the verdict never says hacking and reward_hacking_gap is null" + ] +} diff --git a/changelog.d/941-long-horizon-bench.md b/changelog.d/941-long-horizon-bench.md new file mode 100644 index 00000000..f3b84cce --- /dev/null +++ b/changelog.d/941-long-horizon-bench.md @@ -0,0 +1,9 @@ +Added +A long-horizon bench (`python -m bench.long_horizon`) evolves one playbook +across 5–15 sequential steps on a git ledger, with a deterministic offline +proposer and `DefaultAcceptance` gating each rewrite on held-out checks. It +reports retention, per-check forgetting (mean, max, fraction), paired transfer +on a fresh check set, and cost, alongside the refusals as well as the commits. +On its 3-seed run every seed ends at the domain's reachable ceiling and is +still a `forgetting` verdict, because two check families were erased on the +way (#941). diff --git a/docs/long-horizon-bench.md b/docs/long-horizon-bench.md new file mode 100644 index 00000000..8ab12e87 --- /dev/null +++ b/docs/long-horizon-bench.md @@ -0,0 +1,140 @@ +# Long-horizon bench — one artifact, nine steps, with lineage + +*Script:* [`bench/long_horizon.py`](https://github.com/ThinkFlowLab/agentdescent/blob/main/bench/long_horizon.py) + +Every other bench here measures one hop: a generation loop over a fixed task +set (`bench.rsi_bench`), or each rung of the meta stack run once +(`bench.self_evolution`). Recursive self-improvement is a long loop — the same +artifact rewritten step after step, each step conditioned on the version the +previous one committed — and the usual failure is a decision that was locally +correct at step 4 and wrong at step 12. This bench evolves **one** playbook +across sequential steps on a git ledger, and publishes the refusals as well as +the commits. + +```bash +python -m bench.long_horizon # 9 steps × 3 seeds, human-readable +python -m bench.long_horizon --json # machine-readable report +python -m bench.long_horizon --steps 15 --seeds 3 \ + --out bench/results/long-horizon-3seed.json # the committed results +``` + +It is offline, deterministic and needs no model key. It exits `0` only when +every step was recorded `ok`. + +## The domain + +The artifact is a playbook `SKILL.md` of rule lines. **A later line for the +same rule overrides the earlier one**, so the playbook keeps its history +without being able to hold two answers to the same question — which is what +makes a rewrite able to destroy a skill that was passing. Six check families +read the rules, and they conflict by construction: + +| Family | Checks need | Conflicts with | +| --- | --- | --- | +| `add_sum` | `add: sum` | `add_product` | +| `add_product` | `add: product` | `add_sum`, `chain_add` | +| `mul_sum` | `mul: sum` | `mul_product`, `chain_mul` | +| `mul_product` | `mul: product` | `mul_sum` | +| `chain_add` | `chain: add_then_mul` **and** `add: sum` | `chain_mul`, `add_product` | +| `chain_mul` | `chain: mul_then_add` **and** `mul: product` | `chain_add`, `mul_sum` | + +A chain check scores the share of its two requirements that are in force, so a +chain whose order is right but whose dependency was rewritten away scores one +half. Three checks per family, eighteen in all. No assignment of rule bodies +satisfies every check: the best one reaches **0.583**, and the report prints +that ceiling beside the measured retention so a shortfall can be attributed. + +## One step + +1. The next family in the **seed-shuffled arrival order** becomes the target. +2. A deterministic, offline proposer reads the playbook and emits the single + rule line that fixes that family — no model call. It sees only its target + family. +3. The candidate goes through `FileTree.to_diff`, so the tree's own size and + path guards apply and the diff id is content-fingerprinted. +4. `DefaultAcceptance` gates the rewrite on the **held-out** checks. +5. Accepted → `Ledger.commit`, with the rewrite in the message; refused → the + step is recorded with the gate's own reason and the artifact is untouched. + +Lineage is the chain of committed states. Each step records the content sha of +the playbook before and after it, so the lineage is verifiable without +depending on a git hash — git commit hashes never enter the report. + +## What it measured + +Nine steps, three seeds (`bench/results/long-horizon-3seed.json`, manifest +`455feebf…`): + +| Seed | Retention | Forgetting (max) | Transfer, 95% CI | Committed / refused | Verdict | +| --- | --- | --- | --- | --- | --- | +| 0 | 0.583 | 0.333 (1.000) | +0.583 [+0.333, +0.833] | 5 / 4 | `forgetting` | +| 1 | 0.583 | 0.167 (1.000) | +0.583 [+0.333, +0.833] | 4 / 5 | `forgetting` | +| 2 | 0.583 | 0.333 (1.000) | +0.583 [+0.333, +0.833] | 5 / 4 | `forgetting` | + +Every seed ends on the best reachable assignment (retention 0.583 = the +ceiling), and every seed is still a `forgetting` result: two families, three +checks each, were passing earlier in the run and are gone at the end. Seed 0 +sets `add: product` at step 3 and takes it back at step 4; `mul: sum` from step +2 does not survive `mul: product` at step 6. Aggregate retention hides that +completely — which is why forgetting is reported per check, with a mean, a +maximum and a fraction, exactly as `RSIBenchReport` reports it. + +The other half of the story is in the refusals: 13 of 27 steps were refused. +Seed 0 step 8 re-proposes `mul: sum` and the gate answers +`held-out regression 0.583 -> 0.500`; seed 1 step 5 proposes `chain: +add_then_mul`, which completes `chain_add` but breaks `chain_mul` by the same +half-check, so the posterior lands on `P(delta>0)=0.50` and refuses. On this +domain the gate is what stops the churn from being worse than it is. + +Three seeds are a **range, not significance**: each seed only shuffles which +family arrives when, and the report gives the min and max across them. + +## The four metrics + +Each is defined once, here, and nowhere else. + +| Metric | Definition | +| --- | --- | +| **retention** | the share of all 18 checks the final artifact passes | +| **forgetting** | per check, `max(0, max(valid) - valid[-1])` over the checkpoints after it was first measured — `TaskCurve.forgetting`, reused rather than re-derived; reported as mean, max and fraction | +| **transfer** | final minus step 0 on a fresh check set no step ever measured, paired per check and bootstrapped within the seed (`bootstrap_ci`), then min/max across seeds | +| **cost** | check evaluations, ledger commits, artifact bytes/lines and wall clock | + +On this domain a fresh check asks the same rule question under a new id — a +different instance of a rule body cannot disagree with it — so transfer equals +retention here, and what the number actually shows is the paired step-0 → final +change (+0.583, interval entirely above zero). A domain whose instances have +variable answers would separate the two; the report carries both, so the +coincidence is visible rather than hidden. + +## The gate's blind spot + +The gate reads held-out checks only. The default holds out one check per +family (6 of 18), so every family is represented. Lower it and that stops being +true: `--holdout-fraction 0.25` holds out four checks and the report says +`families_without_holdout: [mul_sum, mul_product]` — a rewrite that costs those +families is invisible to the gate, and the bench still records the loss. That +case is pinned by `tests/test_long_horizon_bench.py`, because a forgetting +metric that cannot fire is worse than no metric. + +## Verdicts + +The vocabulary is `bench.rsi_bench`'s: `insufficient`, `errored`, `hacking`, +`forgetting`, `no-transfer`, `improving`, `regressing`, `flat`. `flat` and +`regressing` are published as verdicts, never omitted. `hacking` cannot fire +here: the domain has no proxy/oracle pair, so `reward_hacking_gap` is `null` +and the report says why. + +A step that raises is recorded as `ok: false` with the exception and the loop +continues — `--fail-step K` forces that path. `all_ok` is false and the process +exits `1`, but the report is complete: a long run that dies on step 12 has no +evidence. + +## Determinism + +Seeds shuffle the arrival order and nothing else; the held-out split is by +stable hash and is the same for every seed. `build_manifest` hashes the spec, +the seeds, the code version and the bench parameters, and two runs of the same +inputs produce the same `manifest_id` and the same report — everything except +`cost.wall_clock_s`, which a test removes before comparing. The temporary +ledgers differ between runs; the measurements do not. diff --git a/mkdocs.yml b/mkdocs.yml index 613911ab..304a005c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -212,6 +212,7 @@ nav: - Longitudinal RSI report: rsi.md - The meta stack on the CLI: meta-cli.md - Self-evolution bench — every rung, end to end: self-evolution-bench.md + - Long-horizon bench — one artifact, many steps, with lineage: long-horizon-bench.md - Verified seeding — gate on trust: verified-seed.md - Joint multi-slot evolution: joint-meta.md - Solver-challenger self-play: selfplay.md diff --git a/tests/test_long_horizon_bench.py b/tests/test_long_horizon_bench.py new file mode 100644 index 00000000..d29703ca --- /dev/null +++ b/tests/test_long_horizon_bench.py @@ -0,0 +1,136 @@ +"""The long-horizon bench has to survive a long run and report what went wrong. + +Its job is to evolve **one** artifact across sequential steps and publish the +step-by-step record. Three fixtures make that trustworthy: + +* the offline ``main(["--steps", "3", "--seeds", "1", "--json"])`` run finishes + inside five seconds and every step is recorded ``ok``; +* a step that raises is recorded as ``ok: false`` with the exception in it, and + the run still finishes -- a long loop that dies on step 12 has no evidence; +* two runs of the same inputs share a ``manifest_id``, so the report identifies + the run rather than the machine. + +The rest pin the two ways a forgetting metric can be vacuous: forgetting must be +computed from a curve that really peaked and fell, and the held-out split must +be *reported* rather than assumed -- a gate that cannot see a family is the case +this bench exists to measure. +""" + +import json +import time + +from bench.long_horizon import ( + FAMILIES, + append_rule, + check_tasks, + main, + parse_playbook, + run_bench, +) + + +def test_the_bench_runs_every_step(capsys): + started = time.perf_counter() + code = main(["--steps", "3", "--seeds", "1", "--json"]) + elapsed = time.perf_counter() - started + assert code == 0 + assert elapsed < 5.0, f"the offline bench took {elapsed:.1f}s" + out = json.loads(capsys.readouterr().out) + + assert out["bench"] == "long_horizon" + assert out["all_ok"] is True + assert out["steps"] == 3 + assert out["seeds"] == [0] + run = out["runs"][0] + assert [step["ok"] for step in run["steps"]] == [True, True, True] + assert len(run["steps"]) == 3 + # every step is either a commit or a refusal, never neither + assert run["committed"] + run["refused"] == 3 + # the checks the gate could see, named rather than assumed + assert out["checks"]["holdout"] == 6 + assert out["checks"]["families_without_holdout"] == [] + assert set(out["checks"]["holdout_by_family"]) == set(FAMILIES) + # retention can never beat the best assignment the domain allows + assert run["retention"] <= out["checks"]["ceiling"] + 1e-12 + assert run["final_rules"] + + +def test_a_raising_step_is_recorded_not_raised(capsys): + code = main(["--steps", "3", "--seeds", "1", "--fail-step", "2", "--json"]) + assert code == 1 + out = json.loads(capsys.readouterr().out) + assert out["all_ok"] is False + run = out["runs"][0] + assert len(run["steps"]) == 3, "the run must finish after a failed step" + failed = run["steps"][1] + assert failed["ok"] is False + assert "injected failure at step 2" in failed["error"] + # the artifact was left exactly as it was, and the loop carried on + assert failed["parent_sha"] == failed["child_sha"] + assert run["steps"][2]["ok"] is True + assert run["verdict"] == "errored" + + +def test_two_runs_share_a_manifest_id_and_every_number(): + first = run_bench(seeds=1, steps=3, code_version="test") + second = run_bench(seeds=1, steps=3, code_version="test") + assert first["manifest_id"] == second["manifest_id"] + for report in (first, second): + report["runs"][0]["cost"].pop("wall_clock_s") + assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) + + +def test_forgetting_is_measured_from_a_curve_that_really_fell(): + report = run_bench(seeds=1, steps=9, code_version="test") + run = report["runs"][0] + assert run["forgetting"] > 0.0 + curves = {task["task_id"]: task for task in run["report"]["tasks"]} + fell = [task for task in curves.values() + if task["forgetting"] > 0.0] + assert fell, "forgetting must come from a per-check curve, not a constant" + for task in fell: + valid = [score for score, error in zip(task["scores"], task["errors"]) + if error is None] + assert max(valid) > valid[-1] + assert task["forgetting"] == max(0.0, max(valid) - valid[-1]) + assert run["max_forgetting"] == max(t["forgetting"] for t in curves.values()) + assert run["verdict"] == "forgetting" + + +def test_a_held_out_split_that_misses_a_family_is_reported(): + report = run_bench(seeds=1, steps=9, holdout_fraction=0.25, + code_version="test") + assert report["checks"]["holdout"] == 4 + assert report["checks"]["families_without_holdout"], ( + "a four-check holdout cannot cover six families; the report must say so") + run = report["runs"][0] + # the gate could not see those families, and the loop still lost checks + assert run["forgetting"] > 0.0 + assert run["verdict"] == "forgetting" + + +def test_a_refused_step_leaves_the_artifact_alone(): + report = run_bench(seeds=1, steps=9, code_version="test") + run = report["runs"][0] + refused = [step for step in run["steps"] if not step["committed"]] + assert refused, "the gate must refuse something on a conflicted domain" + for step in refused: + assert step["ok"] is True + assert step["parent_sha"] == step["child_sha"] + assert step["version_before"] == step["version_after"] + assert step["reason"] + for step in run["lineage"]: + assert step["parent_sha"] != step["child_sha"] + + +def test_a_later_rule_line_overrides_the_earlier_one(): + text = append_rule("# Playbook\n", "add", "sum") + assert parse_playbook(text) == {"add": "sum"} + text = append_rule(text, "add", "product") + # both lines are still in the file; only the last one is in force + assert text.count("- add: ") == 2 + assert parse_playbook(text) == {"add": "product"} + # a header line is not a rule + assert parse_playbook("# Playbook\n(empty)") == {} + # and a check set is stable across calls, so the split is too + assert check_tasks() == check_tasks() From 7194e1b32064097052100e9d89ee06aee8cbe6b7 Mon Sep 17 00:00:00 2001 From: AgentDescent Dev-4 Date: Sat, 21 Nov 2026 19:30:46 +0000 Subject: [PATCH 2/3] refactor(bench): drive the long-horizon loop through step() The issue's scoping comment specifies the shape of a step: `step(artifact, step_no, budget) -> (artifact, BenchStep)`. The loop was inline, so that contract could not be read or tested on its own, and a reader had to infer which numbers a step was responsible for. The step is now a closure over the artifacts it needs and returns the updated artifact plus a BenchStep whose score is the phase family's mean on that checkpoint and whose cost is the checks it evaluated; a raising step comes back as a BenchStep carrying the error instead of unwinding the run. The loop is a three-line driver. No measurement changed: identical manifest id, identical per-step decisions and identical retention/forgetting/transfer, verified by diffing the regenerated results file against the previous one. The new test pins the contract: the recorded score equals the phase family's mean on that checkpoint in the per-check curves, so the metric stays defined in one place. --- bench/long_horizon.py | 149 ++++++++++-------- bench/results/long-horizon-3seed.json | 211 ++++++++++++++++++++------ tests/test_long_horizon_bench.py | 20 +++ 3 files changed, 271 insertions(+), 109 deletions(-) diff --git a/bench/long_horizon.py b/bench/long_horizon.py index 9cee7351..020678ae 100644 --- a/bench/long_horizon.py +++ b/bench/long_horizon.py @@ -95,6 +95,7 @@ from agentdescent.stats_eval import bootstrap_ci from bench.rsi_bench import ( BenchmarkTask, + BenchStep, RSIBench, RSIBenchReport, TaskCurve, @@ -335,74 +336,92 @@ def run_seed(seed: int, *, steps: int = 9, ledger.register(artifact) commits += 1 # the registration is the genesis node of the lineage - try: - for step_no in range(1, steps + 1): - phase = order[(step_no - 1) % len(order)] - effective = parse_playbook(state[PLAYBOOK]) - rule, body = propose(phase, effective) - new_text = append_rule(state[PLAYBOOK], rule, body) - candidate_state = dict(state) - candidate_state[PLAYBOOK] = new_text - proposal = json.dumps({ - "rationale": f"{phase} fails; set {rule}: {body}", - "edits": [{"path": PLAYBOOK, "content": new_text}], + def step(current: EvolvingArtifact, step_no: int, budget: float + ) -> Tuple[EvolvingArtifact, BenchStep]: + """One sequential rewrite, as ``(artifact, BenchStep)``; never a raise. + + The contract the issue asks for: the loop below is only a driver, so the + step can be read, tested and swapped on its own. A failing step comes + back as a ``BenchStep`` carrying the error, which is what lets a + 15-step run that breaks at step 12 still report eleven steps of + evidence. + """ + nonlocal state, evaluations, commits + phase = order[(step_no - 1) % len(order)] + effective = parse_playbook(state[PLAYBOOK]) + rule, body = propose(phase, effective) + new_text = append_rule(state[PLAYBOOK], rule, body) + candidate_state = dict(state) + candidate_state[PLAYBOOK] = new_text + proposal = json.dumps({ + "rationale": f"{phase} fails; set {rule}: {body}", + "edits": [{"path": PLAYBOOK, "content": new_text}], + }) + diff = tree.to_diff(state, proposal, "scripted-proposer", + current.version, current.id) + record: Dict[str, Any] = { + "step": step_no, "phase": phase, "rule": rule, "body": body, + "budget": budget, "ok": True, "error": None, "committed": False, + "parent_sha": content_sha(state), "child_sha": content_sha(state), + "version_before": current.version, "version_after": current.version, + "holdout_before": 0.0, "holdout_after": 0.0, + "p_improve": 0.0, "gate": "", "reason": "", + } + updated = current + try: + if fail_step == step_no: + raise RuntimeError(f"injected failure at step {step_no}") + base_counts = counts(holdout_set, effective) + candidate_effective = parse_playbook(new_text) + cand_counts = counts(holdout_set, candidate_effective) + candidate = EvolvingArtifact(current.id, candidate_state, + version=current.version + 1, + strategy=tree) + before = MergeContext.rate(base_counts) + after = MergeContext.rate(cand_counts) + decision = gate.accept(MergeContext( + artifact=current, candidate=candidate, cards=(), + base_counts=base_counts, cand_counts=cand_counts, diff=diff)) + record.update({ + "holdout_before": before, "holdout_after": after, + "p_improve": decision.p_improve, + "gate": decision.category, + "reason": decision.detail or decision.category, }) - diff = tree.to_diff(state, proposal, "scripted-proposer", - artifact.version, artifact.id) - record: Dict[str, Any] = { - "step": step_no, "phase": phase, "rule": rule, "body": body, - "ok": True, "error": None, "committed": False, - "parent_sha": content_sha(state), "child_sha": content_sha(state), - "version_before": artifact.version, "version_after": artifact.version, - "holdout_before": 0.0, "holdout_after": 0.0, - "p_improve": 0.0, "gate": "", "reason": "", - } - try: - if fail_step == step_no: - raise RuntimeError(f"injected failure at step {step_no}") - base_counts = counts(holdout_set, effective) - evaluations += len(holdout_set) - candidate_effective = parse_playbook(new_text) - cand_counts = counts(holdout_set, candidate_effective) - evaluations += len(holdout_set) - candidate = EvolvingArtifact(artifact.id, candidate_state, - version=artifact.version + 1, - strategy=tree) - before = MergeContext.rate(base_counts) - after = MergeContext.rate(cand_counts) - decision = gate.accept(MergeContext( - artifact=artifact, candidate=candidate, cards=(), - base_counts=base_counts, cand_counts=cand_counts, diff=diff)) + if decision.accept: + head = ledger.head_version() + _, version = ledger.commit( + candidate, head, + message=(f"step {step_no}: {phase} -> {rule}: {body} " + f"(held-out {before:.2f} -> {after:.2f})")) + commits += 1 + updated = EvolvingArtifact(current.id, candidate_state, + version=version, strategy=tree) + state = candidate_state record.update({ - "holdout_before": before, "holdout_after": after, - "p_improve": decision.p_improve, - "gate": decision.category, - "reason": decision.detail or decision.category, + "committed": True, "child_sha": content_sha(state), + "version_after": version, }) - if decision.accept: - head = ledger.head_version() - _, version = ledger.commit( - candidate, head, - message=(f"step {step_no}: {phase} -> {rule}: {body} " - f"(held-out {before:.2f} -> {after:.2f})")) - commits += 1 - artifact = EvolvingArtifact(artifact.id, candidate_state, - version=version, strategy=tree) - state = candidate_state - record.update({ - "committed": True, "child_sha": content_sha(state), - "version_after": version, - }) - except Exception as exc: # noqa: BLE001 - a failed step is a data point - record.update({"ok": False, "gate": "raised", - "error": f"{type(exc).__name__}: {exc}", - "reason": "step raised; state left unchanged"}) - effective = parse_playbook(state[PLAYBOOK]) - step_scores = score_map(tasks, effective) - evaluations += len(tasks) - history.append(step_scores) - errors.append(record["error"]) - records.append(record) + except Exception as exc: # noqa: BLE001 - a failed step is a data point + record.update({"ok": False, "gate": "raised", + "error": f"{type(exc).__name__}: {exc}", + "reason": "step raised; state left unchanged"}) + spent = 2 * len(holdout_set) + len(tasks) + evaluations += spent + step_scores = score_map(tasks, parse_playbook(state[PLAYBOOK])) + history.append(step_scores) + errors.append(record["error"]) + phase_score = _mean([step_scores[task.task_id] for task in tasks + if task.family == phase]) + record.update({"score": phase_score, "cost": float(spent)}) + records.append(record) + return updated, BenchStep(score=phase_score, cost=float(spent), + error=record["error"]) + + step_budget = 1.0 + try: + for step_no in range(1, steps + 1): + artifact, _ = step(artifact, step_no, step_budget) finally: ledger.close() diff --git a/bench/results/long-horizon-3seed.json b/bench/results/long-horizon-3seed.json index 2c48280f..5a05d6e1 100644 --- a/bench/results/long-horizon-3seed.json +++ b/bench/results/long-horizon-3seed.json @@ -71,7 +71,7 @@ "ledger_commits": 6, "artifact_bytes": 210, "artifact_lines": 5, - "wall_clock_s": 0.556 + "wall_clock_s": 0.892 }, "final_rules": { "add": "sum", @@ -84,6 +84,7 @@ "phase": "chain_add", "rule": "chain", "body": "add_then_mul", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -95,13 +96,16 @@ "holdout_after": 0.08333333333333333, "p_improve": 0.652, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 0.5, + "cost": 30.0 }, { "step": 2, "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -113,13 +117,16 @@ "holdout_after": 0.25, "p_improve": 0.73925, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 3, "phase": "add_product", "rule": "add", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -131,13 +138,16 @@ "holdout_after": 0.4166666666666667, "p_improve": 0.707, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 4, "phase": "add_sum", "rule": "add", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -149,13 +159,16 @@ "holdout_after": 0.5, "p_improve": 0.606, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 5, "phase": "chain_mul", "rule": "chain", "body": "mul_then_add", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -167,13 +180,16 @@ "holdout_after": 0.5, "p_improve": 0.5105, "gate": "below-threshold", - "reason": "P(delta>0)=0.51 <= 0.53" + "reason": "P(delta>0)=0.51 <= 0.53", + "score": 0.0, + "cost": 30.0 }, { "step": 6, "phase": "mul_product", "rule": "mul", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -185,13 +201,16 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.6, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 7, "phase": "chain_add", "rule": "chain", "body": "add_then_mul", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -203,13 +222,16 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.4925, "gate": "below-threshold", - "reason": "P(delta>0)=0.49 <= 0.53" + "reason": "P(delta>0)=0.49 <= 0.53", + "score": 1.0, + "cost": 30.0 }, { "step": 8, "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -221,13 +243,16 @@ "holdout_after": 0.5, "p_improve": 0.38475, "gate": "below-threshold", - "reason": "held-out regression 0.583 -> 0.500" + "reason": "held-out regression 0.583 -> 0.500", + "score": 0.0, + "cost": 30.0 }, { "step": 9, "phase": "add_product", "rule": "add", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -239,7 +264,9 @@ "holdout_after": 0.5, "p_improve": 0.3925, "gate": "below-threshold", - "reason": "held-out regression 0.583 -> 0.500" + "reason": "held-out regression 0.583 -> 0.500", + "score": 0.0, + "cost": 30.0 } ], "lineage": [ @@ -248,6 +275,7 @@ "phase": "chain_add", "rule": "chain", "body": "add_then_mul", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -259,13 +287,16 @@ "holdout_after": 0.08333333333333333, "p_improve": 0.652, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 0.5, + "cost": 30.0 }, { "step": 2, "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -277,13 +308,16 @@ "holdout_after": 0.25, "p_improve": 0.73925, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 3, "phase": "add_product", "rule": "add", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -295,13 +329,16 @@ "holdout_after": 0.4166666666666667, "p_improve": 0.707, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 4, "phase": "add_sum", "rule": "add", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -313,13 +350,16 @@ "holdout_after": 0.5, "p_improve": 0.606, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 6, "phase": "mul_product", "rule": "mul", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -331,7 +371,9 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.6, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 } ], "report": { @@ -1376,7 +1418,7 @@ "ledger_commits": 5, "artifact_bytes": 195, "artifact_lines": 4, - "wall_clock_s": 0.501 + "wall_clock_s": 0.637 }, "final_rules": { "add": "sum", @@ -1389,6 +1431,7 @@ "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -1400,13 +1443,16 @@ "holdout_after": 0.16666666666666666, "p_improve": 0.775, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 2, "phase": "mul_product", "rule": "mul", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -1418,13 +1464,16 @@ "holdout_after": 0.25, "p_improve": 0.633, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 3, "phase": "chain_mul", "rule": "chain", "body": "mul_then_add", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -1436,13 +1485,16 @@ "holdout_after": 0.3333333333333333, "p_improve": 0.61675, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 4, "phase": "add_sum", "rule": "add", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -1454,13 +1506,16 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.79275, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 5, "phase": "chain_add", "rule": "chain", "body": "add_then_mul", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -1472,13 +1527,16 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.5035, "gate": "below-threshold", - "reason": "P(delta>0)=0.50 <= 0.53" + "reason": "P(delta>0)=0.50 <= 0.53", + "score": 0.5, + "cost": 30.0 }, { "step": 6, "phase": "add_product", "rule": "add", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -1490,13 +1548,16 @@ "holdout_after": 0.5, "p_improve": 0.4055, "gate": "below-threshold", - "reason": "held-out regression 0.583 -> 0.500" + "reason": "held-out regression 0.583 -> 0.500", + "score": 0.0, + "cost": 30.0 }, { "step": 7, "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -1508,13 +1569,16 @@ "holdout_after": 0.5, "p_improve": 0.39, "gate": "below-threshold", - "reason": "held-out regression 0.583 -> 0.500" + "reason": "held-out regression 0.583 -> 0.500", + "score": 0.0, + "cost": 30.0 }, { "step": 8, "phase": "mul_product", "rule": "mul", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -1526,13 +1590,16 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.50625, "gate": "below-threshold", - "reason": "P(delta>0)=0.51 <= 0.53" + "reason": "P(delta>0)=0.51 <= 0.53", + "score": 1.0, + "cost": 30.0 }, { "step": 9, "phase": "chain_mul", "rule": "chain", "body": "mul_then_add", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -1544,7 +1611,9 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.4985, "gate": "below-threshold", - "reason": "P(delta>0)=0.50 <= 0.53" + "reason": "P(delta>0)=0.50 <= 0.53", + "score": 1.0, + "cost": 30.0 } ], "lineage": [ @@ -1553,6 +1622,7 @@ "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -1564,13 +1634,16 @@ "holdout_after": 0.16666666666666666, "p_improve": 0.775, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 2, "phase": "mul_product", "rule": "mul", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -1582,13 +1655,16 @@ "holdout_after": 0.25, "p_improve": 0.633, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 3, "phase": "chain_mul", "rule": "chain", "body": "mul_then_add", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -1600,13 +1676,16 @@ "holdout_after": 0.3333333333333333, "p_improve": 0.61675, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 4, "phase": "add_sum", "rule": "add", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -1618,7 +1697,9 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.79275, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 } ], "report": { @@ -2663,7 +2744,7 @@ "ledger_commits": 6, "artifact_bytes": 210, "artifact_lines": 5, - "wall_clock_s": 0.595 + "wall_clock_s": 0.569 }, "final_rules": { "add": "sum", @@ -2676,6 +2757,7 @@ "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2687,13 +2769,16 @@ "holdout_after": 0.16666666666666666, "p_improve": 0.775, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 2, "phase": "mul_product", "rule": "mul", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2705,13 +2790,16 @@ "holdout_after": 0.25, "p_improve": 0.633, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 3, "phase": "add_product", "rule": "add", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2723,13 +2811,16 @@ "holdout_after": 0.4166666666666667, "p_improve": 0.70675, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 4, "phase": "chain_add", "rule": "chain", "body": "add_then_mul", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2741,13 +2832,16 @@ "holdout_after": 0.5, "p_improve": 0.60525, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 0.5, + "cost": 30.0 }, { "step": 5, "phase": "chain_mul", "rule": "chain", "body": "mul_then_add", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -2759,13 +2853,16 @@ "holdout_after": 0.5, "p_improve": 0.4975, "gate": "below-threshold", - "reason": "P(delta>0)=0.50 <= 0.53" + "reason": "P(delta>0)=0.50 <= 0.53", + "score": 0.5, + "cost": 30.0 }, { "step": 6, "phase": "add_sum", "rule": "add", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2777,13 +2874,16 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.60575, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 7, "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -2795,13 +2895,16 @@ "holdout_after": 0.5, "p_improve": 0.38475, "gate": "below-threshold", - "reason": "held-out regression 0.583 -> 0.500" + "reason": "held-out regression 0.583 -> 0.500", + "score": 0.0, + "cost": 30.0 }, { "step": 8, "phase": "mul_product", "rule": "mul", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -2813,13 +2916,16 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.50025, "gate": "below-threshold", - "reason": "P(delta>0)=0.50 <= 0.53" + "reason": "P(delta>0)=0.50 <= 0.53", + "score": 1.0, + "cost": 30.0 }, { "step": 9, "phase": "add_product", "rule": "add", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": false, @@ -2831,7 +2937,9 @@ "holdout_after": 0.5, "p_improve": 0.3925, "gate": "below-threshold", - "reason": "held-out regression 0.583 -> 0.500" + "reason": "held-out regression 0.583 -> 0.500", + "score": 0.0, + "cost": 30.0 } ], "lineage": [ @@ -2840,6 +2948,7 @@ "phase": "mul_sum", "rule": "mul", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2851,13 +2960,16 @@ "holdout_after": 0.16666666666666666, "p_improve": 0.775, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 2, "phase": "mul_product", "rule": "mul", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2869,13 +2981,16 @@ "holdout_after": 0.25, "p_improve": 0.633, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 3, "phase": "add_product", "rule": "add", "body": "product", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2887,13 +3002,16 @@ "holdout_after": 0.4166666666666667, "p_improve": 0.70675, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 }, { "step": 4, "phase": "chain_add", "rule": "chain", "body": "add_then_mul", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2905,13 +3023,16 @@ "holdout_after": 0.5, "p_improve": 0.60525, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 0.5, + "cost": 30.0 }, { "step": 6, "phase": "add_sum", "rule": "add", "body": "sum", + "budget": 1.0, "ok": true, "error": null, "committed": true, @@ -2923,7 +3044,9 @@ "holdout_after": 0.5833333333333334, "p_improve": 0.60575, "gate": "committed", - "reason": "committed" + "reason": "committed", + "score": 1.0, + "cost": 30.0 } ], "report": { diff --git a/tests/test_long_horizon_bench.py b/tests/test_long_horizon_bench.py index d29703ca..e7871774 100644 --- a/tests/test_long_horizon_bench.py +++ b/tests/test_long_horizon_bench.py @@ -19,6 +19,8 @@ import json import time +import pytest + from bench.long_horizon import ( FAMILIES, append_rule, @@ -109,6 +111,24 @@ def test_a_held_out_split_that_misses_a_family_is_reported(): assert run["verdict"] == "forgetting" +def test_each_step_returns_a_bench_step_for_its_phase(): + report = run_bench(seeds=1, steps=4, code_version="test") + run = report["runs"][0] + curves = run["report"]["tasks"] + for record in run["steps"]: + # a BenchStep: a finite score, a non-negative cost, an error slot + assert 0.0 <= record["score"] <= 1.0 + assert record["cost"] > 0 + assert record["budget"] > 0 + # the score is the phase family's mean on that checkpoint, so the + # metric is defined in one place and the step cannot invent its own + family = [task for task in curves if task["family"] == record["phase"]] + assert family + mean = sum(task["scores"][record["step"]] for task in family) / len(family) + assert record["score"] == pytest.approx(mean) + assert len({record["phase"] for record in run["steps"]}) == 4 + + def test_a_refused_step_leaves_the_artifact_alone(): report = run_bench(seeds=1, steps=9, code_version="test") run = report["runs"][0] From be265c88065620bb933284a994a2d7fc24463290 Mon Sep 17 00:00:00 2001 From: AgentDescent Dev-4 Date: Sat, 21 Nov 2026 19:32:57 +0000 Subject: [PATCH 3/3] docs(bench): use the committed nine-step command in the long-horizon page --- docs/long-horizon-bench.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/long-horizon-bench.md b/docs/long-horizon-bench.md index 8ab12e87..bf5a58d6 100644 --- a/docs/long-horizon-bench.md +++ b/docs/long-horizon-bench.md @@ -14,7 +14,7 @@ the commits. ```bash python -m bench.long_horizon # 9 steps × 3 seeds, human-readable python -m bench.long_horizon --json # machine-readable report -python -m bench.long_horizon --steps 15 --seeds 3 \ +python -m bench.long_horizon --steps 9 --seeds 3 \ --out bench/results/long-horizon-3seed.json # the committed results ```