From 667b0a51fb1dfed56f03230618b6ddc3d3ce12d5 Mon Sep 17 00:00:00 2001 From: Teakowa <27560638+Teakowa@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:53:40 +0800 Subject: [PATCH 1/2] feat(bench): re-grade stored runs when a scenario's checks change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #614 A changed scenario check turns the grader hash over, but stored runs kept their old verdict because nothing wrote new grades back. agent_bench.py regrade --out copies every trial into a run-shaped output directory — inputs are never modified — and re-grades only the trials whose recorded grader hash differs, recording the earlier hash as grader.regradedFrom. Trials at the current hash are copied without grading. report --regrade also no longer leaves regrade-*/ scratch directories inside the run directories; both paths grade into scratch outside the archived runs. --- benchmarks/agent/agent_bench.py | 9 +- benchmarks/agent/bench_regrade.py | 124 +++++++++++++++++++++++++++ benchmarks/agent/bench_report.py | 24 ++++-- benchmarks/agent/test_agent_bench.py | 90 +++++++++++++++++++ docs/agent-benchmark-howto.md | 19 ++++ 5 files changed, 256 insertions(+), 10 deletions(-) create mode 100644 benchmarks/agent/bench_regrade.py diff --git a/benchmarks/agent/agent_bench.py b/benchmarks/agent/agent_bench.py index 8ce50919..3cd92a73 100644 --- a/benchmarks/agent/agent_bench.py +++ b/benchmarks/agent/agent_bench.py @@ -26,6 +26,7 @@ import bench_grade import bench_leaderboard +import bench_regrade import bench_report import bench_publish import bench_score @@ -1126,13 +1127,17 @@ def build_parser() -> argparse.ArgumentParser: report.add_argument("--out-dir", type=Path, help="where report.md and summary.json go; with several run directories they are only printed unless this is given") report.add_argument("--regrade", action="store_true", help="re-grade stored workspaces twice and flag unstable graders") report.add_argument("--wright", default=str(ROOT / "target/debug/wright")) + regrade_cmd = sub.add_parser("regrade", help="copy run directories and re-grade trials whose recorded grader hash differs from the current scenario") + regrade_cmd.add_argument("dirs", nargs="+", type=Path) + regrade_cmd.add_argument("--wright", default=str(ROOT / "target/debug/wright")) + regrade_cmd.add_argument("--out", type=Path, required=True, help="new directory holding the re-graded copies; input directories are never modified") report.add_argument("--reference", action="append", help="condition label a paired comparison is made against; repeatable for lift against several references (default: none/none/off)") compare = sub.add_parser("compare", help="one table from the score.json of several evaluation runs, warning when they are not comparable") compare.add_argument("dirs", nargs="+", type=Path) score = sub.add_parser("score", help="compute the Wright Agent Score card of each language track from canonical test runs") score.add_argument("dirs", nargs="+", type=Path) score.add_argument("--language", choices=("workshop", "opy"), action="append", help="track to score; both when omitted") - for name in ("validate", "run", "matrix", "evaluate", "suite", "track", "report", "score"): + for name in ("validate", "run", "matrix", "evaluate", "suite", "track", "report", "regrade", "score"): sub.choices[name].add_argument("--private-suite", type=Path, help="private versioned suite directory, kept unreadable by the agent") defaults = user_defaults() for choice in sub.choices.values(): @@ -1169,6 +1174,8 @@ def main() -> int: return cmd_wiki_skill(args) if args.command == "report": return bench_report.main(args.dirs, args.wright, args.regrade, lambda s: load_scenario(s, args.private_suite), references_of(args), args.out_dir) + if args.command == "regrade": + return bench_regrade.main(args.dirs, args.wright, lambda s: load_scenario(s, args.private_suite), args.out) if args.command == "publish": try: return bench_publish.main(args.dirs, args.out, args.dry_run, args.endpoint) diff --git a/benchmarks/agent/bench_regrade.py b/benchmarks/agent/bench_regrade.py new file mode 100644 index 00000000..f12052fd --- /dev/null +++ b/benchmarks/agent/bench_regrade.py @@ -0,0 +1,124 @@ +"""Re-grade stored benchmark trials when a scenario's checks change (#614, SPEC-414 REQ-007). + +A changed `scenario.json` or grader source changes the grader hash, so a run +directory recorded under an earlier hash no longer reflects the current +verdict. `regrade` copies each trial into an output directory — originals are +never written — and re-grades only the trials whose recorded hash differs. +""" + +from __future__ import annotations + +import shutil +import tempfile +from pathlib import Path + +import bench_grade +import bench_report + +GRADER_UNAVAILABLE = "a required grader was unavailable" + + +def status_of(result: dict) -> str: + """`agent_bench.run_status` without the import cycle: the agent outcome, not the verdict.""" + if "invalid" in result: + return "invalid" + code = result["agent"]["exit"] + if code == 75: + return "provider-interrupted" + if code is None: + return "timeout" + return "agent-error" if code else "completed" + + +def regrade_trial(result: dict, dest: Path, scenario: dict, wright: str) -> bool: + """Copy the stored trial into `dest` and re-grade it when the grader hash changed. + + The copy carries every field the agent produced untouched; only the fields + `grade` owns are replaced. Returns True when the trial was re-graded. + """ + source = result["_dir"] + shutil.copytree(source, dest) + current = bench_grade.grader_hash(scenario) + recorded = (result.get("grader") or {}).get("hash") + if recorded == current: + return False + workspace = dest / "workspace" + if not workspace.is_dir(): + return False + usable_before = result.get("usable") + verdict = bench_grade.grade(scenario, workspace, wright, dest / "grading") + result.update(verdict) + result["grader"]["regradedFrom"] = recorded + # The agent-side invalid reasons cannot be re-derived, but a grader that + # was unavailable then and resolves now stops being a blocking reason. + if "invalid" in result and not any(c.get("unavailable") for c in result["checks"]): + reasons = [r for r in result["invalid"].split("; ") if r != GRADER_UNAVAILABLE] + if reasons != result["invalid"].split("; "): + if reasons: + result["invalid"] = "; ".join(reasons) + else: + del result["invalid"] + result["status"] = status_of(result) + result["_verdictChanged"] = usable_before != result.get("usable") + bench_report.write_json(dest / "result.json", {k: v for k, v in result.items() if not k.startswith("_")}) + return True + + +def regrade(dirs: list[Path], wright: str, load_scenario, out: Path) -> dict: + """Copy every trial under `dirs` into `out`, re-grading hash-mismatched trials. + + `out` is laid out as `//` so the result is + consumable by `report`, `score`, and `publish` exactly like a run + directory. No input directory is modified. + """ + resolved = [base.resolve() for base in dirs] + out = out.resolve() + for base in resolved: + if base == out or out in base.parents or base in out.parents: + raise SystemExit("--out must be disjoint from every input directory") + if out.exists() and any(out.iterdir()): + raise SystemExit(f"--out {out} already exists and is not empty") + scenarios: dict[str, dict] = {} + graded = copied = changed = 0 + transitions: dict[str, set] = {} + for base in resolved: + for result in bench_report.load([base]): + scenario_id = result["scenario"] + if scenario_id not in scenarios: + scenarios[scenario_id] = load_scenario(scenario_id) + dest = out / base.name / result["_dir"].relative_to(base) + dest.parent.mkdir(parents=True, exist_ok=True) + if regrade_trial(result, dest, scenarios[scenario_id], wright): + graded += 1 + changed += bool(result.pop("_verdictChanged", False)) + grader = result["grader"] + transitions.setdefault(scenario_id, set()).add((grader["regradedFrom"], grader["hash"])) + else: + copied += 1 + lines = [] + for scenario_id in sorted(transitions): + for old, new in sorted(transitions[scenario_id]): + lines.append(f"{scenario_id}: grader {old[:12]} -> {new[:12]}") + summary = { + "trials": graded + copied, + "copied": copied, + "regraded": graded, + "verdictChanged": changed, + "out": str(out), + "transitions": lines, + } + return summary + + +def main(dirs: list[Path], wright: str, load_scenario, out: Path) -> int: + summary = regrade(dirs, wright, load_scenario, out) + if summary["regraded"] == 0: + print(f"every recorded grader hash is current; {summary['copied']} trial(s) copied to {summary['out']}") + else: + for line in summary["transitions"]: + print(f"REGRADED {line}") + print( + f"{summary['regraded']} trial(s) re-graded, {summary['verdictChanged']} usable verdict(s) changed, " + f"{summary['copied']} copied unchanged -> {summary['out']}" + ) + return 0 diff --git a/benchmarks/agent/bench_report.py b/benchmarks/agent/bench_report.py index d73660b7..0a2a519c 100644 --- a/benchmarks/agent/bench_report.py +++ b/benchmarks/agent/bench_report.py @@ -274,17 +274,23 @@ def diagnostics(runs: list[dict], invalid: list[dict]) -> list[str]: def regrade_notes(runs: list[dict], wright: str, load_scenario) -> list[str]: - """Grader consistency: grading the same stored workspace twice must give one verdict.""" + """Grader consistency: grading the same stored workspace twice must give one verdict. + The scratch directories live in a temporary directory: a run directory is an archive nothing writes into (#614).""" import bench_grade + import tempfile unstable = [] - for r in runs: - workspace = r["_dir"] / "workspace" - if not workspace.is_dir(): - continue - scenario = load_scenario(r["scenario"]) - first, second = (bench_grade.grade(scenario, workspace, wright, r["_dir"] / f"regrade-{i}") for i in (1, 2)) - if [(c["id"], c["passed"]) for c in first["checks"]] != [(c["id"], c["passed"]) for c in second["checks"]]: - unstable.append(str(r["_dir"])) + with tempfile.TemporaryDirectory(prefix="wright-bench-regrade-") as scratch: + for index, r in enumerate(runs): + workspace = r["_dir"] / "workspace" + if not workspace.is_dir(): + continue + scenario = load_scenario(r["scenario"]) + first, second = ( + bench_grade.grade(scenario, workspace, wright, Path(scratch) / f"{index}-{i}") + for i in (1, 2) + ) + if [(c["id"], c["passed"]) for c in first["checks"]] != [(c["id"], c["passed"]) for c in second["checks"]]: + unstable.append(str(r["_dir"])) return [f"GRADER: unstable verdict on {len(unstable)} workspace(s): {unstable}"] if unstable else ["GRADER: consistent on every regraded workspace."] diff --git a/benchmarks/agent/test_agent_bench.py b/benchmarks/agent/test_agent_bench.py index 90b4ef6c..a37fc3c3 100644 --- a/benchmarks/agent/test_agent_bench.py +++ b/benchmarks/agent/test_agent_bench.py @@ -15,6 +15,7 @@ import agent_bench import bench_grade +import bench_regrade import bench_report import bench_trace import wiki_skill @@ -982,6 +983,95 @@ def test_oracle_disagreement_is_reported(self): self.assertEqual(auth["disagreement"]["kind"], "wright-accepts-oracle-rejects") +@unittest.skipUnless(Path(WRIGHT).is_file(), "build wright first or set WRIGHT_BIN") +class RegradeTest(unittest.TestCase): + """#614: `regrade` re-grades stored trials whose scenario checks changed.""" + + WORKSPACE = ( + 'variables {\n global:\n 0: counter\n}\n\n' + 'rule ("tick") {\n event {\n Ongoing - Global;\n }\n' + ' actions {\n Set Global Variable(counter, 1);\n }\n}\n' + ) + + def setUp(self): + (agent_bench.ROOT / "target").mkdir(exist_ok=True) + self.out = Path(tempfile.mkdtemp(dir=agent_bench.ROOT / "target")).resolve() + self.addCleanup(shutil.rmtree, self.out, True) + self.scenarios = self.out / "scenarios" / "tiny" + self.write_scenario("counter") + + def write_scenario(self, wanted: str) -> Path: + (self.scenarios / "seed").mkdir(parents=True, exist_ok=True) + (self.scenarios / "seed" / "mode.ws").write_text(self.WORKSPACE) + spec = { + "canary": agent_bench.CANARY, "id": "tiny", "family": "modification", "language": "workshop", + "entry": "mode.ws", "writable": ["mode.ws"], + "checks": [{"id": "has-text", "kind": "contains", "file": "mode.ws", "text": wanted}], + } + (self.scenarios / "scenario.json").write_text(json.dumps(spec)) + return self.scenarios + + def scenario(self, _scenario_id: str) -> dict: + spec = json.loads((self.scenarios / "scenario.json").read_text()) + return {**spec, "dir": self.scenarios} + + def stored_trial(self, name: str, grader_hash: str, usable: bool = True) -> Path: + trial = self.out / "runs" / "tiny" / "fake" / f"none-none_off-{name}" + (trial / "workspace").mkdir(parents=True) + (trial / "workspace" / "mode.ws").write_text(self.WORKSPACE) + (trial / "agent.log").write_text("exit=0\n") + result = { + "contract": agent_bench.RESULT_CONTRACT, "status": "completed", "language": "workshop", + "condition": {"tool": "none", "level": "bin", "skills": [], "knowledge": "none", "network": "off", "label": "none/none/off"}, + "scenario": "tiny", "agent": {"id": "fake", "command": "true", "exit": 0, "seconds": 0.1}, + "environment": {"timestamp": "2026-10-11T00:00:00+00:00"}, + "passed": True, "usable": usable, + "checks": [{"id": "has-text", "kind": "contains", "passed": True, "detail": "1 occurrence(s) in mode.ws"}], + "grader": {"hash": grader_hash, "oracle": None}, + } + (trial / "result.json").write_text(json.dumps(result)) + return trial + + def test_a_changed_check_regrades_the_copy_and_preserves_the_original(self): + current = bench_grade.grader_hash(self.scenario("tiny")) + trial = self.stored_trial("1", current, usable=True) + original_result = (trial / "result.json").read_text() + self.write_scenario("no-such-marker") # the check change turns the grader hash over + summary = bench_regrade.regrade([self.out / "runs"], WRIGHT, self.scenario, self.out / "regraded") + self.assertEqual((summary["regraded"], summary["verdictChanged"], summary["copied"]), (1, 1, 0)) + self.assertEqual((trial / "result.json").read_text(), original_result) # the input run is never written + copy = self.out / "regraded" / "runs" / "tiny" / "fake" / "none-none_off-1" + result = json.loads((copy / "result.json").read_text()) + self.assertFalse(result["usable"]) + self.assertFalse(result["checks"][0]["passed"]) + self.assertEqual(result["grader"]["regradedFrom"], current) + self.assertEqual(result["grader"]["hash"], bench_grade.grader_hash(self.scenario("tiny"))) + self.assertEqual((copy / "agent.log").read_text(), "exit=0\n") # what the grader did not produce is preserved + self.assertEqual(result["agent"]["id"], "fake") + + def test_a_current_hash_is_copied_without_grading(self): + current = bench_grade.grader_hash(self.scenario("tiny")) + self.stored_trial("1", current) + summary = bench_regrade.regrade([self.out / "runs"], WRIGHT, self.scenario, self.out / "regraded") + self.assertEqual((summary["regraded"], summary["copied"]), (0, 1)) + copy = self.out / "regraded" / "runs" / "tiny" / "fake" / "none-none_off-1" + result = json.loads((copy / "result.json").read_text()) + self.assertNotIn("regradedFrom", result["grader"]) + self.assertFalse((copy / "grading").exists(), "a matching hash never invokes the grader") + + def test_report_regrade_leaves_no_scratch_inside_the_run_directory(self): + current = bench_grade.grader_hash(self.scenario("tiny")) + trial = self.stored_trial("1", current) + runs = bench_report.load([self.out / "runs"]) + notes = bench_report.regrade_notes(runs, WRIGHT, self.scenario) + self.assertTrue(notes and "GRADER" in notes[0]) + self.assertEqual([p for p in trial.rglob("regrade-*")], []) + self.assertEqual( + sorted(p.name for p in trial.iterdir()), + ["agent.log", "result.json", "workspace"], + ) + + @unittest.skipUnless(sys.platform == "darwin", "file flags are the macOS enforcement") class DropTest(unittest.TestCase): def setUp(self): diff --git a/docs/agent-benchmark-howto.md b/docs/agent-benchmark-howto.md index 6acb1f08..1bfd40ce 100644 --- a/docs/agent-benchmark-howto.md +++ b/docs/agent-benchmark-howto.md @@ -105,6 +105,25 @@ python3 benchmarks/agent/agent_bench.py report ~/.local/share/wright-agent-bench A report over several run directories is printed and written only where `--out-dir` says, so it never replaces a run's own report. +## Re-grade runs after a scenario's checks change + +Fixing a task's checks changes its grader hash, and a run recorded under the +old hash keeps its old verdict until it is re-graded. Re-grade after editing a +scenario or the grader, before reporting on older run directories: + +```sh +python3 benchmarks/agent/agent_bench.py regrade ~/.local/share/wright-agent-bench/runs/ --out +``` + +Every trial is copied under `--out` — the input run directories are never +written — and only the trials whose recorded grader hash differs are graded +again with the current scenario and `--wright` binary. A re-graded +`result.json` keeps everything the agent produced (transcript, trace, usage) +and records the earlier hash as `grader.regradedFrom`. Point `report`, +`score`, `leaderboard`, `compare`, or `publish` at `--out` exactly as at a run +directory. `report --regrade` still only checks grader stability; it never +writes verdicts back. + ## Publish the hosted results Maintainers publish the data the public results page reads (see From 652de1ecec46e0a87789c2d764d670bf2493c6d5 Mon Sep 17 00:00:00 2001 From: Teakowa <27560638+Teakowa@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:54:31 +0800 Subject: [PATCH 2/2] fix(bench): re-grade pre-hash archives and share the run-status rule Refs #614 Archives written before grader.hash existed carry no recorded hash, so regrade_trial set regradedFrom=None and regrade() then crashed building the transition summary (sorted() on (None, str) tuples, and old[:12] on None). Pre-hash trials now re-grade cleanly, keep regradedFrom: null in the copied result, and read in the transition report; a regression test covers the grader-less archive path. run_status and INFRA_EXIT move to bench_report so agent_bench and bench_regrade share one definition instead of a duplicated status_of with a hardcoded 75; the unused tempfile import goes with it. --- benchmarks/agent/agent_bench.py | 16 ++-------------- benchmarks/agent/bench_regrade.py | 19 ++++--------------- benchmarks/agent/bench_report.py | 13 +++++++++++++ benchmarks/agent/test_agent_bench.py | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/benchmarks/agent/agent_bench.py b/benchmarks/agent/agent_bench.py index 3cd92a73..8a95ca70 100644 --- a/benchmarks/agent/agent_bench.py +++ b/benchmarks/agent/agent_bench.py @@ -43,7 +43,7 @@ SKILLS = ("wright-skill", "workshop-skill", "opy-skill", "workshop-format-skill") SKILL_LANGUAGE = {"opy-skill": "opy", "workshop-format-skill": "workshop"} # skills that teach one language and apply to its scenarios only KNOWLEDGE_LEVELS = ("none", "wiki", "web") -INFRA_EXIT = 75 # EX_TEMPFAIL: the adapter reports a provider or infrastructure failure, not an agent failure +INFRA_EXIT = bench_report.INFRA_EXIT # EX_TEMPFAIL: the adapter reports a provider or infrastructure failure, not an agent failure ENV_KEEP = ("LANG", "LC_ALL", "TERM", "TMPDIR", "USER", "LOGNAME") @@ -545,23 +545,11 @@ def run_trial(scenario: dict, cell: dict, args: argparse.Namespace, out: Path) - result["snapshots"] = snapshot_validity(scenario, snaps, args.wright, out) first_valid = next((s["t"] for s in result["snapshots"]["series"] if s["valid"]), None) result["usage"] = bench_trace.usage_summary(out / "usage.jsonl", first_valid) - result["status"] = run_status(result) + result["status"] = bench_report.run_status(result) bench_report.write_json(out / "result.json", result) return result -def run_status(result: dict) -> str: - """How the run ended, separate from whether its artifact was usable.""" - if "invalid" in result: - return "invalid" - code = result["agent"]["exit"] - if code == INFRA_EXIT: - return "provider-interrupted" - if code is None: - return "timeout" - return "agent-error" if code else "completed" - - def snapshot_validity(scenario: dict, snaps: list[dict], wright: str, out: Path) -> dict: """Strict validity of every snapshot of the entry file; regressions are valid -> invalid transitions.""" series = [] diff --git a/benchmarks/agent/bench_regrade.py b/benchmarks/agent/bench_regrade.py index f12052fd..9e8e425c 100644 --- a/benchmarks/agent/bench_regrade.py +++ b/benchmarks/agent/bench_regrade.py @@ -9,7 +9,6 @@ from __future__ import annotations import shutil -import tempfile from pathlib import Path import bench_grade @@ -18,18 +17,6 @@ GRADER_UNAVAILABLE = "a required grader was unavailable" -def status_of(result: dict) -> str: - """`agent_bench.run_status` without the import cycle: the agent outcome, not the verdict.""" - if "invalid" in result: - return "invalid" - code = result["agent"]["exit"] - if code == 75: - return "provider-interrupted" - if code is None: - return "timeout" - return "agent-error" if code else "completed" - - def regrade_trial(result: dict, dest: Path, scenario: dict, wright: str) -> bool: """Copy the stored trial into `dest` and re-grade it when the grader hash changed. @@ -58,7 +45,7 @@ def regrade_trial(result: dict, dest: Path, scenario: dict, wright: str) -> bool result["invalid"] = "; ".join(reasons) else: del result["invalid"] - result["status"] = status_of(result) + result["status"] = bench_report.run_status(result) result["_verdictChanged"] = usable_before != result.get("usable") bench_report.write_json(dest / "result.json", {k: v for k, v in result.items() if not k.startswith("_")}) return True @@ -92,7 +79,9 @@ def regrade(dirs: list[Path], wright: str, load_scenario, out: Path) -> dict: graded += 1 changed += bool(result.pop("_verdictChanged", False)) grader = result["grader"] - transitions.setdefault(scenario_id, set()).add((grader["regradedFrom"], grader["hash"])) + # Pre-hash archives have no recorded hash — `regradedFrom` + # stays `null` in the result and reads `` here. + transitions.setdefault(scenario_id, set()).add((grader["regradedFrom"] or "", grader["hash"])) else: copied += 1 lines = [] diff --git a/benchmarks/agent/bench_report.py b/benchmarks/agent/bench_report.py index 0a2a519c..03d28998 100644 --- a/benchmarks/agent/bench_report.py +++ b/benchmarks/agent/bench_report.py @@ -12,6 +12,19 @@ BASELINE = "none/none/off" HEADROOM = 0.95 +INFRA_EXIT = 75 # EX_TEMPFAIL: the adapter reports a provider or infrastructure failure, not an agent failure + + +def run_status(result: dict) -> str: + """How the run ended, separate from whether its artifact was usable.""" + if "invalid" in result: + return "invalid" + code = result["agent"]["exit"] + if code == INFRA_EXIT: + return "provider-interrupted" + if code is None: + return "timeout" + return "agent-error" if code else "completed" def write_json(path: Path, data: dict) -> None: diff --git a/benchmarks/agent/test_agent_bench.py b/benchmarks/agent/test_agent_bench.py index a37fc3c3..502cbc9a 100644 --- a/benchmarks/agent/test_agent_bench.py +++ b/benchmarks/agent/test_agent_bench.py @@ -1059,6 +1059,24 @@ def test_a_current_hash_is_copied_without_grading(self): self.assertNotIn("regradedFrom", result["grader"]) self.assertFalse((copy / "grading").exists(), "a matching hash never invokes the grader") + def test_a_pre_hash_archive_regrades_and_reports_the_transition(self): + # Archives written before `grader.hash` existed carry no recorded + # hash: every such trial is stale and must re-grade, and the + # transition summary reads `` instead of crashing (#621). + trial = self.stored_trial("1", "unrelated") + stored = json.loads((trial / "result.json").read_text()) + del stored["grader"] + (trial / "result.json").write_text(json.dumps(stored)) + self.write_scenario("no-such-marker") + summary = bench_regrade.regrade([self.out / "runs"], WRIGHT, self.scenario, self.out / "regraded") + self.assertEqual(summary["regraded"], 1) + new_hash = bench_grade.grader_hash(self.scenario("tiny")) + self.assertEqual(summary["transitions"], [f"tiny: grader -> {new_hash[:12]}"]) + copy = self.out / "regraded" / "runs" / "tiny" / "fake" / "none-none_off-1" + result = json.loads((copy / "result.json").read_text()) + self.assertIsNone(result["grader"]["regradedFrom"]) + self.assertEqual(result["grader"]["hash"], new_hash) + def test_report_regrade_leaves_no_scratch_inside_the_run_directory(self): current = bench_grade.grader_hash(self.scenario("tiny")) trial = self.stored_trial("1", current)