feat(bench): re-grade stored runs when a scenario's checks change

benchmarks/agent/agent_bench.py

@@ -26,6 +26,7 @@
 
 import bench_grade
 import bench_leaderboard
+import bench_regrade
 import bench_report
 import bench_publish
 import bench_score
@@ -42,7 +43,7 @@
 SKILLS = ("wright-skill", "workshop-skill", "opy-skill", "workshop-format-skill")
 SKILL_LANGUAGE = {"opy-skill": "opy", "workshop-format-skill": "workshop"}  # skills that teach one language and apply to its scenarios only
 KNOWLEDGE_LEVELS = ("none", "wiki", "web")
-INFRA_EXIT = 75  # EX_TEMPFAIL: the adapter reports a provider or infrastructure failure, not an agent failure
+INFRA_EXIT = bench_report.INFRA_EXIT  # EX_TEMPFAIL: the adapter reports a provider or infrastructure failure, not an agent failure
 ENV_KEEP = ("LANG", "LC_ALL", "TERM", "TMPDIR", "USER", "LOGNAME")
 
 
@@ -544,23 +545,11 @@ def run_trial(scenario: dict, cell: dict, args: argparse.Namespace, out: Path) -
     result["snapshots"] = snapshot_validity(scenario, snaps, args.wright, out)
     first_valid = next((s["t"] for s in result["snapshots"]["series"] if s["valid"]), None)
     result["usage"] = bench_trace.usage_summary(out / "usage.jsonl", first_valid)
-    result["status"] = run_status(result)
+    result["status"] = bench_report.run_status(result)
     bench_report.write_json(out / "result.json", result)
     return result
 
 
-def run_status(result: dict) -> str:
-    """How the run ended, separate from whether its artifact was usable."""
-    if "invalid" in result:
-        return "invalid"
-    code = result["agent"]["exit"]
-    if code == INFRA_EXIT:
-        return "provider-interrupted"
-    if code is None:
-        return "timeout"
-    return "agent-error" if code else "completed"
-
-
 def snapshot_validity(scenario: dict, snaps: list[dict], wright: str, out: Path) -> dict:
     """Strict validity of every snapshot of the entry file; regressions are valid -> invalid transitions."""
     series = []
@@ -1126,13 +1115,17 @@ def build_parser() -> argparse.ArgumentParser:
     report.add_argument("--out-dir", type=Path, help="where report.md and summary.json go; with several run directories they are only printed unless this is given")
     report.add_argument("--regrade", action="store_true", help="re-grade stored workspaces twice and flag unstable graders")
     report.add_argument("--wright", default=str(ROOT / "target/debug/wright"))
+    regrade_cmd = sub.add_parser("regrade", help="copy run directories and re-grade trials whose recorded grader hash differs from the current scenario")
+    regrade_cmd.add_argument("dirs", nargs="+", type=Path)
+    regrade_cmd.add_argument("--wright", default=str(ROOT / "target/debug/wright"))
+    regrade_cmd.add_argument("--out", type=Path, required=True, help="new directory holding the re-graded copies; input directories are never modified")
     report.add_argument("--reference", action="append", help="condition label a paired comparison is made against; repeatable for lift against several references (default: none/none/off)")
     compare = sub.add_parser("compare", help="one table from the score.json of several evaluation runs, warning when they are not comparable")
     compare.add_argument("dirs", nargs="+", type=Path)
     score = sub.add_parser("score", help="compute the Wright Agent Score card of each language track from canonical test runs")
     score.add_argument("dirs", nargs="+", type=Path)
     score.add_argument("--language", choices=("workshop", "opy"), action="append", help="track to score; both when omitted")
-    for name in ("validate", "run", "matrix", "evaluate", "suite", "track", "report", "score"):
+    for name in ("validate", "run", "matrix", "evaluate", "suite", "track", "report", "regrade", "score"):
         sub.choices[name].add_argument("--private-suite", type=Path, help="private versioned suite directory, kept unreadable by the agent")
     defaults = user_defaults()
     for choice in sub.choices.values():
@@ -1169,6 +1162,8 @@ def main() -> int:
         return cmd_wiki_skill(args)
     if args.command == "report":
         return bench_report.main(args.dirs, args.wright, args.regrade, lambda s: load_scenario(s, args.private_suite), references_of(args), args.out_dir)
+    if args.command == "regrade":
+        return bench_regrade.main(args.dirs, args.wright, lambda s: load_scenario(s, args.private_suite), args.out)
     if args.command == "publish":
         try:
             return bench_publish.main(args.dirs, args.out, args.dry_run, args.endpoint)

benchmarks/agent/bench_regrade.py

@@ -0,0 +1,113 @@
+"""Re-grade stored benchmark trials when a scenario's checks change (#614, SPEC-414 REQ-007).
+
+A changed `scenario.json` or grader source changes the grader hash, so a run
+directory recorded under an earlier hash no longer reflects the current
+verdict. `regrade` copies each trial into an output directory — originals are
+never written — and re-grades only the trials whose recorded hash differs.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import bench_grade
+import bench_report
+
+GRADER_UNAVAILABLE = "a required grader was unavailable"
+
+
+def regrade_trial(result: dict, dest: Path, scenario: dict, wright: str) -> bool:
+    """Copy the stored trial into `dest` and re-grade it when the grader hash changed.
+
+    The copy carries every field the agent produced untouched; only the fields
+    `grade` owns are replaced. Returns True when the trial was re-graded.
+    """
+    source = result["_dir"]
+    shutil.copytree(source, dest)
+    current = bench_grade.grader_hash(scenario)
+    recorded = (result.get("grader") or {}).get("hash")
+    if recorded == current:
+        return False
+    workspace = dest / "workspace"
+    if not workspace.is_dir():
+        return False
+    usable_before = result.get("usable")
+    verdict = bench_grade.grade(scenario, workspace, wright, dest / "grading")
+    result.update(verdict)
+    result["grader"]["regradedFrom"] = recorded
+    # The agent-side invalid reasons cannot be re-derived, but a grader that
+    # was unavailable then and resolves now stops being a blocking reason.
+    if "invalid" in result and not any(c.get("unavailable") for c in result["checks"]):
+        reasons = [r for r in result["invalid"].split("; ") if r != GRADER_UNAVAILABLE]
+        if reasons != result["invalid"].split("; "):
+            if reasons:
+                result["invalid"] = "; ".join(reasons)
+            else:
+                del result["invalid"]
+    result["status"] = bench_report.run_status(result)
+    result["_verdictChanged"] = usable_before != result.get("usable")
+    bench_report.write_json(dest / "result.json", {k: v for k, v in result.items() if not k.startswith("_")})
+    return True
+
+
+def regrade(dirs: list[Path], wright: str, load_scenario, out: Path) -> dict:
+    """Copy every trial under `dirs` into `out`, re-grading hash-mismatched trials.
+
+    `out` is laid out as `<out>/<run-dir-name>/<trial path>` so the result is
+    consumable by `report`, `score`, and `publish` exactly like a run
+    directory. No input directory is modified.
+    """
+    resolved = [base.resolve() for base in dirs]
+    out = out.resolve()
+    for base in resolved:
+        if base == out or out in base.parents or base in out.parents:
+            raise SystemExit("--out must be disjoint from every input directory")
+    if out.exists() and any(out.iterdir()):
+        raise SystemExit(f"--out {out} already exists and is not empty")
+    scenarios: dict[str, dict] = {}
+    graded = copied = changed = 0
+    transitions: dict[str, set] = {}
+    for base in resolved:
+        for result in bench_report.load([base]):
+            scenario_id = result["scenario"]
+            if scenario_id not in scenarios:
+                scenarios[scenario_id] = load_scenario(scenario_id)
+            dest = out / base.name / result["_dir"].relative_to(base)
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            if regrade_trial(result, dest, scenarios[scenario_id], wright):
+                graded += 1
+                changed += bool(result.pop("_verdictChanged", False))
+                grader = result["grader"]
+                # Pre-hash archives have no recorded hash — `regradedFrom`
+                # stays `null` in the result and reads `<none>` here.
+                transitions.setdefault(scenario_id, set()).add((grader["regradedFrom"] or "<none>", grader["hash"]))
+            else:
+                copied += 1
+    lines = []
+    for scenario_id in sorted(transitions):
+        for old, new in sorted(transitions[scenario_id]):
+            lines.append(f"{scenario_id}: grader {old[:12]} -> {new[:12]}")
+    summary = {
+        "trials": graded + copied,
+        "copied": copied,
+        "regraded": graded,
+        "verdictChanged": changed,
+        "out": str(out),
+        "transitions": lines,
+    }
+    return summary
+
+
+def main(dirs: list[Path], wright: str, load_scenario, out: Path) -> int:
+    summary = regrade(dirs, wright, load_scenario, out)
+    if summary["regraded"] == 0:
+        print(f"every recorded grader hash is current; {summary['copied']} trial(s) copied to {summary['out']}")
+    else:
+        for line in summary["transitions"]:
+            print(f"REGRADED {line}")
+        print(
+            f"{summary['regraded']} trial(s) re-graded, {summary['verdictChanged']} usable verdict(s) changed, "
+            f"{summary['copied']} copied unchanged -> {summary['out']}"
+        )
+    return 0

benchmarks/agent/bench_report.py

@@ -12,6 +12,19 @@
 
 BASELINE = "none/none/off"
 HEADROOM = 0.95
+INFRA_EXIT = 75  # EX_TEMPFAIL: the adapter reports a provider or infrastructure failure, not an agent failure
+
+
+def run_status(result: dict) -> str:
+    """How the run ended, separate from whether its artifact was usable."""
+    if "invalid" in result:
+        return "invalid"
+    code = result["agent"]["exit"]
+    if code == INFRA_EXIT:
+        return "provider-interrupted"
+    if code is None:
+        return "timeout"
+    return "agent-error" if code else "completed"
 
 
 def write_json(path: Path, data: dict) -> None:
@@ -274,17 +287,23 @@ def diagnostics(runs: list[dict], invalid: list[dict]) -> list[str]:
 
 
 def regrade_notes(runs: list[dict], wright: str, load_scenario) -> list[str]:
-    """Grader consistency: grading the same stored workspace twice must give one verdict."""
+    """Grader consistency: grading the same stored workspace twice must give one verdict.
+    The scratch directories live in a temporary directory: a run directory is an archive nothing writes into (#614)."""
     import bench_grade
+    import tempfile
     unstable = []
-    for r in runs:
-        workspace = r["_dir"] / "workspace"
-        if not workspace.is_dir():
-            continue
-        scenario = load_scenario(r["scenario"])
-        first, second = (bench_grade.grade(scenario, workspace, wright, r["_dir"] / f"regrade-{i}") for i in (1, 2))
-        if [(c["id"], c["passed"]) for c in first["checks"]] != [(c["id"], c["passed"]) for c in second["checks"]]:
-            unstable.append(str(r["_dir"]))
+    with tempfile.TemporaryDirectory(prefix="wright-bench-regrade-") as scratch:
+        for index, r in enumerate(runs):
+            workspace = r["_dir"] / "workspace"
+            if not workspace.is_dir():
+                continue
+            scenario = load_scenario(r["scenario"])
+            first, second = (
+                bench_grade.grade(scenario, workspace, wright, Path(scratch) / f"{index}-{i}")
+                for i in (1, 2)
+            )
+            if [(c["id"], c["passed"]) for c in first["checks"]] != [(c["id"], c["passed"]) for c in second["checks"]]:
+                unstable.append(str(r["_dir"]))
     return [f"GRADER: unstable verdict on {len(unstable)} workspace(s): {unstable}"] if unstable else ["GRADER: consistent on every regraded workspace."]
 
 

benchmarks/agent/test_agent_bench.py

@@ -15,6 +15,7 @@
 
 import agent_bench
 import bench_grade
+import bench_regrade
 import bench_report
 import bench_trace
 import wiki_skill
@@ -982,6 +983,113 @@ def test_oracle_disagreement_is_reported(self):
             self.assertEqual(auth["disagreement"]["kind"], "wright-accepts-oracle-rejects")
 
 
+@unittest.skipUnless(Path(WRIGHT).is_file(), "build wright first or set WRIGHT_BIN")
+class RegradeTest(unittest.TestCase):
+    """#614: `regrade` re-grades stored trials whose scenario checks changed."""
+
+    WORKSPACE = (
+        'variables {\n    global:\n        0: counter\n}\n\n'
+        'rule ("tick") {\n    event {\n        Ongoing - Global;\n    }\n'
+        '    actions {\n        Set Global Variable(counter, 1);\n    }\n}\n'
+    )
+
+    def setUp(self):
+        (agent_bench.ROOT / "target").mkdir(exist_ok=True)
+        self.out = Path(tempfile.mkdtemp(dir=agent_bench.ROOT / "target")).resolve()
+        self.addCleanup(shutil.rmtree, self.out, True)
+        self.scenarios = self.out / "scenarios" / "tiny"
+        self.write_scenario("counter")
+
+    def write_scenario(self, wanted: str) -> Path:
+        (self.scenarios / "seed").mkdir(parents=True, exist_ok=True)
+        (self.scenarios / "seed" / "mode.ws").write_text(self.WORKSPACE)
+        spec = {
+            "canary": agent_bench.CANARY, "id": "tiny", "family": "modification", "language": "workshop",
+            "entry": "mode.ws", "writable": ["mode.ws"],
+            "checks": [{"id": "has-text", "kind": "contains", "file": "mode.ws", "text": wanted}],
+        }
+        (self.scenarios / "scenario.json").write_text(json.dumps(spec))
+        return self.scenarios
+
+    def scenario(self, _scenario_id: str) -> dict:
+        spec = json.loads((self.scenarios / "scenario.json").read_text())
+        return {**spec, "dir": self.scenarios}
+
+    def stored_trial(self, name: str, grader_hash: str, usable: bool = True) -> Path:
+        trial = self.out / "runs" / "tiny" / "fake" / f"none-none_off-{name}"
+        (trial / "workspace").mkdir(parents=True)
+        (trial / "workspace" / "mode.ws").write_text(self.WORKSPACE)
+        (trial / "agent.log").write_text("exit=0\n")
+        result = {
+            "contract": agent_bench.RESULT_CONTRACT, "status": "completed", "language": "workshop",
+            "condition": {"tool": "none", "level": "bin", "skills": [], "knowledge": "none", "network": "off", "label": "none/none/off"},
+            "scenario": "tiny", "agent": {"id": "fake", "command": "true", "exit": 0, "seconds": 0.1},
+            "environment": {"timestamp": "2026-10-11T00:00:00+00:00"},
+            "passed": True, "usable": usable,
+            "checks": [{"id": "has-text", "kind": "contains", "passed": True, "detail": "1 occurrence(s) in mode.ws"}],
+            "grader": {"hash": grader_hash, "oracle": None},
+        }
+        (trial / "result.json").write_text(json.dumps(result))
+        return trial
+
+    def test_a_changed_check_regrades_the_copy_and_preserves_the_original(self):
+        current = bench_grade.grader_hash(self.scenario("tiny"))
+        trial = self.stored_trial("1", current, usable=True)
+        original_result = (trial / "result.json").read_text()
+        self.write_scenario("no-such-marker")  # the check change turns the grader hash over
+        summary = bench_regrade.regrade([self.out / "runs"], WRIGHT, self.scenario, self.out / "regraded")
+        self.assertEqual((summary["regraded"], summary["verdictChanged"], summary["copied"]), (1, 1, 0))
+        self.assertEqual((trial / "result.json").read_text(), original_result)  # the input run is never written
+        copy = self.out / "regraded" / "runs" / "tiny" / "fake" / "none-none_off-1"
+        result = json.loads((copy / "result.json").read_text())
+        self.assertFalse(result["usable"])
+        self.assertFalse(result["checks"][0]["passed"])
+        self.assertEqual(result["grader"]["regradedFrom"], current)
+        self.assertEqual(result["grader"]["hash"], bench_grade.grader_hash(self.scenario("tiny")))
+        self.assertEqual((copy / "agent.log").read_text(), "exit=0\n")  # what the grader did not produce is preserved
+        self.assertEqual(result["agent"]["id"], "fake")
+
+    def test_a_current_hash_is_copied_without_grading(self):
+        current = bench_grade.grader_hash(self.scenario("tiny"))
+        self.stored_trial("1", current)
+        summary = bench_regrade.regrade([self.out / "runs"], WRIGHT, self.scenario, self.out / "regraded")
+        self.assertEqual((summary["regraded"], summary["copied"]), (0, 1))
+        copy = self.out / "regraded" / "runs" / "tiny" / "fake" / "none-none_off-1"
+        result = json.loads((copy / "result.json").read_text())
+        self.assertNotIn("regradedFrom", result["grader"])
+        self.assertFalse((copy / "grading").exists(), "a matching hash never invokes the grader")
+
+    def test_a_pre_hash_archive_regrades_and_reports_the_transition(self):
+        # Archives written before `grader.hash` existed carry no recorded
+        # hash: every such trial is stale and must re-grade, and the
+        # transition summary reads `<none>` instead of crashing (#621).
+        trial = self.stored_trial("1", "unrelated")
+        stored = json.loads((trial / "result.json").read_text())
+        del stored["grader"]
+        (trial / "result.json").write_text(json.dumps(stored))
+        self.write_scenario("no-such-marker")
+        summary = bench_regrade.regrade([self.out / "runs"], WRIGHT, self.scenario, self.out / "regraded")
+        self.assertEqual(summary["regraded"], 1)
+        new_hash = bench_grade.grader_hash(self.scenario("tiny"))
+        self.assertEqual(summary["transitions"], [f"tiny: grader <none> -> {new_hash[:12]}"])
+        copy = self.out / "regraded" / "runs" / "tiny" / "fake" / "none-none_off-1"
+        result = json.loads((copy / "result.json").read_text())
+        self.assertIsNone(result["grader"]["regradedFrom"])
+        self.assertEqual(result["grader"]["hash"], new_hash)
+
+    def test_report_regrade_leaves_no_scratch_inside_the_run_directory(self):
+        current = bench_grade.grader_hash(self.scenario("tiny"))
+        trial = self.stored_trial("1", current)
+        runs = bench_report.load([self.out / "runs"])
+        notes = bench_report.regrade_notes(runs, WRIGHT, self.scenario)
+        self.assertTrue(notes and "GRADER" in notes[0])
+        self.assertEqual([p for p in trial.rglob("regrade-*")], [])
+        self.assertEqual(
+            sorted(p.name for p in trial.iterdir()),
+            ["agent.log", "result.json", "workspace"],
+        )
+
+
 @unittest.skipUnless(sys.platform == "darwin", "file flags are the macOS enforcement")
 class DropTest(unittest.TestCase):
     def setUp(self):

docs/agent-benchmark-howto.md

@@ -105,6 +105,25 @@ python3 benchmarks/agent/agent_bench.py report ~/.local/share/wright-agent-bench
 
 A report over several run directories is printed and written only where `--out-dir` says, so it never replaces a run's own report.
 
+## Re-grade runs after a scenario's checks change
+
+Fixing a task's checks changes its grader hash, and a run recorded under the
+old hash keeps its old verdict until it is re-graded. Re-grade after editing a
+scenario or the grader, before reporting on older run directories:
+
+```sh
+python3 benchmarks/agent/agent_bench.py regrade ~/.local/share/wright-agent-bench/runs/<run> --out <new-dir>
+```
+
+Every trial is copied under `--out` — the input run directories are never
+written — and only the trials whose recorded grader hash differs are graded
+again with the current scenario and `--wright` binary. A re-graded
+`result.json` keeps everything the agent produced (transcript, trace, usage)
+and records the earlier hash as `grader.regradedFrom`. Point `report`,
+`score`, `leaderboard`, `compare`, or `publish` at `--out` exactly as at a run
+directory. `report --regrade` still only checks grader stability; it never
+writes verdicts back.
+
 ## Publish the hosted results
 
 Maintainers publish the data the public results page reads (see