diff --git a/app/ui_tests/test_judge_calibration_reads.py b/app/ui_tests/test_judge_calibration_reads.py new file mode 100644 index 00000000..91b49d92 --- /dev/null +++ b/app/ui_tests/test_judge_calibration_reads.py @@ -0,0 +1,63 @@ +"""Judge all reads the judge's calibration once for the batch, not the +whole store again for every trial it judges.""" +from __future__ import annotations + +import builtins +import os +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from types import SimpleNamespace as NS +from unittest.mock import patch + +from nullius import workspace +from nullius.judge import questions as Q + +from .test_routes import web + + +def rec(*a, **k): + return {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"}, + "usage": {"input_tokens": 100}, "states": {"trial": {"chars": 300}}} + + +class CalibrationReads(unittest.TestCase): + def setUp(self): + stack = ExitStack() + self.addCleanup(stack.close) + tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-judge-calib-"))) + stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces"))) + stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True)) + stack.enter_context(patch.dict(os.environ, TYPESAFE_API_KEY="ts-fake")) + stack.enter_context(patch.object(web.judge.batch, "ask_states", side_effect=rec)) + stack.enter_context(patch.object(web.judge.trajectory, "load", lambda path: NS(error=None))) + stack.enter_context(patch.object(web.judge, "_trial_ids_and_states", lambda trial, traj: {})) + self.client = web.app.test_client() + with self.client.session_transaction() as sess: + sess["csrf"] = "t" + + def reads_to_judge_all(self, n): + web.shared.open_workspace(workspace.create(web.shared.WS_HOME, [], f"w{n}")) + job_id = "job" + trials = [NS(trial_name=f"t{i}", has_trajectory=True, trajectory_path="-", started_at=None, outcome="resolved") for i in range(n)] + job = NS(job_id=job_id, trials=trials) + real_open, reads = builtins.open, [0] + + def counting_open(path, mode="r", *a, **k): + if "judgements" in str(path) and "r" in mode: + reads[0] += 1 + return real_open(path, mode, *a, **k) + + with patch.object(web.shared, "_job_for_request", lambda j: job), patch("builtins.open", counting_open): + self.assertEqual(self.client.post(f"/judge/job/{job_id}", data={"csrf": "t"}).status_code, 302) + self.assertEqual(len(web.judge.store.for_job(web.judge.ws_dir(), job_id, {t.trial_name: None for t in trials})), n) + return reads[0] + + def test_store_reads_grow_linearly_with_the_trials_judged(self): + counts = {n: self.reads_to_judge_all(n) for n in (50, 100, 200)} + self.assertLess(counts[200], 4 * 200, counts) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/feasibility.py b/app/web/feasibility.py index 0db3f469..f8087c59 100644 --- a/app/web/feasibility.py +++ b/app/web/feasibility.py @@ -42,9 +42,10 @@ def judge_feasibility_route(job_id): todo = summary["todo"] judge._guard() done, failed = 0, [] + cal = judge.batch_calibration() for row, side in todo: try: - judge.judge_control(side, jobs, judge.key()); done += 1 + judge.judge_control(side, jobs, judge.key(), calibration=cal); done += 1 except client.JudgeError as e: failed.append(f"{row.task_base} ({side.control}): {e}") if e.status in (401, 0): diff --git a/app/web/judge.py b/app/web/judge.py index b7b06196..7edcae59 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -64,6 +64,12 @@ def ws_dir(): return ws.dir if ws else None +def batch_calibration() -> dict: + """The workspace's calibration, read once for a batch and passed to each + call; {} when there is no sample, so no call reads the store again.""" + return store.calibration(ws_dir()) or {} + + def started_of(trial) -> str | None: """A trial's start, as its judgement keeps it: a retry runs an attempt again under the same name, and an earlier try's record is not its own.""" @@ -248,16 +254,17 @@ def _trial_ids_and_states(trial, traj) -> dict: return _states_for(trial, traj, ids) -def judge_trial(job, trial, api_key: str, post=None) -> dict: +def judge_trial(job, trial, api_key: str, post=None, calibration=None) -> dict: """Ask every question the trial page shows, one call per state, and keep - the merged record. Returns the record as stored.""" + the merged record. Returns the record as stored. `calibration`: a + batch's, read once (`store.calibration`); the workspace's when none.""" if not trial.has_trajectory: raise client.JudgeError(0, "this trial has no trajectory to judge") traj = trajectory.load(trial.trajectory_path) if traj.error: raise client.JudgeError(0, "the trajectory could not be read: " + traj.error) return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, _trial_ids_and_states(trial, traj), api_key, post=post, - started=started_of(trial)) + calibration=calibration, started=started_of(trial)) def _guard(): @@ -300,8 +307,8 @@ def judge_job_route(job_id): judged = store.for_job(d, job_id, started_by_name(job), lane=lane) if d else {} todo = [t for t in job.trials if t.has_trajectory and t.trial_name not in judged] _guard() - k = key(lane) - done, failed = run_batch(todo, lambda t: judge_trial(job, t, k), lambda t: t.trial_name) + k, cal = key(lane), batch_calibration() + done, failed = run_batch(todo, lambda t: judge_trial(job, t, k, calibration=cal), lambda t: t.trial_name) from flask import flash flash(f"{'LLM judge' if lane == 'llm' else 'Decision model'}: judged {done} of {len(todo)} trial(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("job", job_id=job_id) + "#judgement") @@ -437,7 +444,7 @@ def compare_summary(view) -> dict | None: "margin": comparison.TIE_MARGIN, "question": SIDE_QUESTION} -def judge_side(job, trial, api_key: str, post=None) -> dict | None: +def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict | None: """Score one side: the questions it still lacks under the current vocabulary, one call per state, merged over the trial's own record - the same record the trial page shows. Nothing when it has them all.""" @@ -453,7 +460,7 @@ def judge_side(job, trial, api_key: str, post=None) -> dict | None: if traj.error: raise client.JudgeError(0, "the trajectory could not be read: " + traj.error) return batch.ask_and_store(d, job.job_id, trial.trial_name, _states_for(trial, traj, ids), api_key, post=post, - started=started_of(trial)) + calibration=calibration, started=started_of(trial)) def _sides_todo(view) -> list: @@ -477,8 +484,8 @@ def judge_compare_route(): sides = _sides_todo(view) todo = list(dict.fromkeys(task for task, _, _ in sides)) _guard() - k = key() - done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})") + k, cal = key(), batch_calibration() + done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})") from flask import flash flash(f"Judged {done} of {len(sides)} side(s) over {len(todo)} pair(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("compare", pick=view.references, cohort=view.cohort, view=compare_web._view()) + "#judgement") @@ -532,14 +539,14 @@ def control_states(side, trial) -> dict: return batch.states_for(Q.group_ids("feasibility"), build) -def judge_control(side, jobs, api_key: str, post=None) -> dict | None: +def judge_control(side, jobs, api_key: str, post=None, calibration=None) -> dict | None: """Ask why one control side did not behave, over its own `control` state, stored as that control trial's record. The verdict is not touched.""" job, trial = _control_trial(side, jobs) if trial is None: raise client.JudgeError(0, f"{side.control} trial {side.trial_name} is not in this workspace") return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, control_states(side, trial), api_key, post=post, - started=started_of(trial)) + calibration=calibration, started=started_of(trial)) def triage_summary(report, jobs) -> dict: @@ -645,11 +652,11 @@ def task_summary(job) -> dict: return {"tasks": len(tasks), "judged": judged, "unjudged": len(tasks_todo(job)), "keyed": bool(key())} -def judge_task(job, trial, api_key: str, post=None) -> dict | None: +def judge_task(job, trial, api_key: str, post=None, calibration=None) -> dict | None: """Score one task's instruction over the `task` state alone, stored under the dataset's task slot and the instruction's digest - never under the trial.""" instruction = _task_instruction(trial) if not instruction: raise client.JudgeError(0, f"no instruction can be read for {trial.task_base}") states = batch.states_for(Q.group_ids("task"), lambda kind: judge_state.build(kind, None, instruction=instruction)) - return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post) + return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post, calibration=calibration) diff --git a/app/web/judge_tasks.py b/app/web/judge_tasks.py index 960bf985..f42d3a84 100644 --- a/app/web/judge_tasks.py +++ b/app/web/judge_tasks.py @@ -21,8 +21,8 @@ def judge_tasks_route(job_id): abort(404) todo = judge.tasks_todo(job) judge._guard() - k = judge.key() - done, failed = judge.run_batch(todo, lambda t: judge.judge_task(job, t, k), lambda t: t.task_base) + k, cal = judge.key(), judge.batch_calibration() + done, failed = judge.run_batch(todo, lambda t: judge.judge_task(job, t, k, calibration=cal), lambda t: t.task_base) flash(f"Judged {done} of {len(todo)} task instruction(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("job", job_id=job_id) + "#task-quality")