From e06034924dc89be1c743324c4f71d72f4a638ed3 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 12:18:32 +0100 Subject: [PATCH] perf(judge): read the judge's calibration once per batch ask_and_store reads the workspace's calibration when its caller gives none, and that read parses every judgement file of the workspace. Judge all, Compare's Judge, the task batch and the feasibility triage gave none, so a batch walked the whole store again before each call: judging 200 trials into an empty workspace read about 19,000 files. Each batch now reads the calibration once (`batch_calibration`) and passes it to every call, `{}` when the store has no sample yet so no call reads it again. A single Judge from the trial page still reads the workspace's own. The same batch of 200 trials reads 400 files. --- app/ui_tests/test_judge_calibration_reads.py | 63 ++++++++++++++++++++ app/web/feasibility.py | 3 +- app/web/judge.py | 33 ++++++---- app/web/judge_tasks.py | 4 +- 4 files changed, 87 insertions(+), 16 deletions(-) create mode 100644 app/ui_tests/test_judge_calibration_reads.py diff --git a/app/ui_tests/test_judge_calibration_reads.py b/app/ui_tests/test_judge_calibration_reads.py new file mode 100644 index 00000000..91b49d92 --- /dev/null +++ b/app/ui_tests/test_judge_calibration_reads.py @@ -0,0 +1,63 @@ +"""Judge all reads the judge's calibration once for the batch, not the +whole store again for every trial it judges.""" +from __future__ import annotations + +import builtins +import os +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from types import SimpleNamespace as NS +from unittest.mock import patch + +from nullius import workspace +from nullius.judge import questions as Q + +from .test_routes import web + + +def rec(*a, **k): + return {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"}, + "usage": {"input_tokens": 100}, "states": {"trial": {"chars": 300}}} + + +class CalibrationReads(unittest.TestCase): + def setUp(self): + stack = ExitStack() + self.addCleanup(stack.close) + tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-judge-calib-"))) + stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces"))) + stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True)) + stack.enter_context(patch.dict(os.environ, TYPESAFE_API_KEY="ts-fake")) + stack.enter_context(patch.object(web.judge.batch, "ask_states", side_effect=rec)) + stack.enter_context(patch.object(web.judge.trajectory, "load", lambda path: NS(error=None))) + stack.enter_context(patch.object(web.judge, "_trial_ids_and_states", lambda trial, traj: {})) + self.client = web.app.test_client() + with self.client.session_transaction() as sess: + sess["csrf"] = "t" + + def reads_to_judge_all(self, n): + web.shared.open_workspace(workspace.create(web.shared.WS_HOME, [], f"w{n}")) + job_id = "job" + trials = [NS(trial_name=f"t{i}", has_trajectory=True, trajectory_path="-", started_at=None, outcome="resolved") for i in range(n)] + job = NS(job_id=job_id, trials=trials) + real_open, reads = builtins.open, [0] + + def counting_open(path, mode="r", *a, **k): + if "judgements" in str(path) and "r" in mode: + reads[0] += 1 + return real_open(path, mode, *a, **k) + + with patch.object(web.shared, "_job_for_request", lambda j: job), patch("builtins.open", counting_open): + self.assertEqual(self.client.post(f"/judge/job/{job_id}", data={"csrf": "t"}).status_code, 302) + self.assertEqual(len(web.judge.store.for_job(web.judge.ws_dir(), job_id, {t.trial_name: None for t in trials})), n) + return reads[0] + + def test_store_reads_grow_linearly_with_the_trials_judged(self): + counts = {n: self.reads_to_judge_all(n) for n in (50, 100, 200)} + self.assertLess(counts[200], 4 * 200, counts) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/feasibility.py b/app/web/feasibility.py index 0db3f469..f8087c59 100644 --- a/app/web/feasibility.py +++ b/app/web/feasibility.py @@ -42,9 +42,10 @@ def judge_feasibility_route(job_id): todo = summary["todo"] judge._guard() done, failed = 0, [] + cal = judge.batch_calibration() for row, side in todo: try: - judge.judge_control(side, jobs, judge.key()); done += 1 + judge.judge_control(side, jobs, judge.key(), calibration=cal); done += 1 except client.JudgeError as e: failed.append(f"{row.task_base} ({side.control}): {e}") if e.status in (401, 0): diff --git a/app/web/judge.py b/app/web/judge.py index b7b06196..7edcae59 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -64,6 +64,12 @@ def ws_dir(): return ws.dir if ws else None +def batch_calibration() -> dict: + """The workspace's calibration, read once for a batch and passed to each + call; {} when there is no sample, so no call reads the store again.""" + return store.calibration(ws_dir()) or {} + + def started_of(trial) -> str | None: """A trial's start, as its judgement keeps it: a retry runs an attempt again under the same name, and an earlier try's record is not its own.""" @@ -248,16 +254,17 @@ def _trial_ids_and_states(trial, traj) -> dict: return _states_for(trial, traj, ids) -def judge_trial(job, trial, api_key: str, post=None) -> dict: +def judge_trial(job, trial, api_key: str, post=None, calibration=None) -> dict: """Ask every question the trial page shows, one call per state, and keep - the merged record. Returns the record as stored.""" + the merged record. Returns the record as stored. `calibration`: a + batch's, read once (`store.calibration`); the workspace's when none.""" if not trial.has_trajectory: raise client.JudgeError(0, "this trial has no trajectory to judge") traj = trajectory.load(trial.trajectory_path) if traj.error: raise client.JudgeError(0, "the trajectory could not be read: " + traj.error) return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, _trial_ids_and_states(trial, traj), api_key, post=post, - started=started_of(trial)) + calibration=calibration, started=started_of(trial)) def _guard(): @@ -300,8 +307,8 @@ def judge_job_route(job_id): judged = store.for_job(d, job_id, started_by_name(job), lane=lane) if d else {} todo = [t for t in job.trials if t.has_trajectory and t.trial_name not in judged] _guard() - k = key(lane) - done, failed = run_batch(todo, lambda t: judge_trial(job, t, k), lambda t: t.trial_name) + k, cal = key(lane), batch_calibration() + done, failed = run_batch(todo, lambda t: judge_trial(job, t, k, calibration=cal), lambda t: t.trial_name) from flask import flash flash(f"{'LLM judge' if lane == 'llm' else 'Decision model'}: judged {done} of {len(todo)} trial(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("job", job_id=job_id) + "#judgement") @@ -437,7 +444,7 @@ def compare_summary(view) -> dict | None: "margin": comparison.TIE_MARGIN, "question": SIDE_QUESTION} -def judge_side(job, trial, api_key: str, post=None) -> dict | None: +def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict | None: """Score one side: the questions it still lacks under the current vocabulary, one call per state, merged over the trial's own record - the same record the trial page shows. Nothing when it has them all.""" @@ -453,7 +460,7 @@ def judge_side(job, trial, api_key: str, post=None) -> dict | None: if traj.error: raise client.JudgeError(0, "the trajectory could not be read: " + traj.error) return batch.ask_and_store(d, job.job_id, trial.trial_name, _states_for(trial, traj, ids), api_key, post=post, - started=started_of(trial)) + calibration=calibration, started=started_of(trial)) def _sides_todo(view) -> list: @@ -477,8 +484,8 @@ def judge_compare_route(): sides = _sides_todo(view) todo = list(dict.fromkeys(task for task, _, _ in sides)) _guard() - k = key() - done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})") + k, cal = key(), batch_calibration() + done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})") from flask import flash flash(f"Judged {done} of {len(sides)} side(s) over {len(todo)} pair(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("compare", pick=view.references, cohort=view.cohort, view=compare_web._view()) + "#judgement") @@ -532,14 +539,14 @@ def control_states(side, trial) -> dict: return batch.states_for(Q.group_ids("feasibility"), build) -def judge_control(side, jobs, api_key: str, post=None) -> dict | None: +def judge_control(side, jobs, api_key: str, post=None, calibration=None) -> dict | None: """Ask why one control side did not behave, over its own `control` state, stored as that control trial's record. The verdict is not touched.""" job, trial = _control_trial(side, jobs) if trial is None: raise client.JudgeError(0, f"{side.control} trial {side.trial_name} is not in this workspace") return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, control_states(side, trial), api_key, post=post, - started=started_of(trial)) + calibration=calibration, started=started_of(trial)) def triage_summary(report, jobs) -> dict: @@ -645,11 +652,11 @@ def task_summary(job) -> dict: return {"tasks": len(tasks), "judged": judged, "unjudged": len(tasks_todo(job)), "keyed": bool(key())} -def judge_task(job, trial, api_key: str, post=None) -> dict | None: +def judge_task(job, trial, api_key: str, post=None, calibration=None) -> dict | None: """Score one task's instruction over the `task` state alone, stored under the dataset's task slot and the instruction's digest - never under the trial.""" instruction = _task_instruction(trial) if not instruction: raise client.JudgeError(0, f"no instruction can be read for {trial.task_base}") states = batch.states_for(Q.group_ids("task"), lambda kind: judge_state.build(kind, None, instruction=instruction)) - return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post) + return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post, calibration=calibration) diff --git a/app/web/judge_tasks.py b/app/web/judge_tasks.py index 960bf985..f42d3a84 100644 --- a/app/web/judge_tasks.py +++ b/app/web/judge_tasks.py @@ -21,8 +21,8 @@ def judge_tasks_route(job_id): abort(404) todo = judge.tasks_todo(job) judge._guard() - k = judge.key() - done, failed = judge.run_batch(todo, lambda t: judge.judge_task(job, t, k), lambda t: t.task_base) + k, cal = judge.key(), judge.batch_calibration() + done, failed = judge.run_batch(todo, lambda t: judge.judge_task(job, t, k, calibration=cal), lambda t: t.task_base) flash(f"Judged {done} of {len(todo)} task instruction(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("job", job_id=job_id) + "#task-quality")