Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions app/ui_tests/test_judge_calibration_reads.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""Judge all reads the judge's calibration once for the batch, not the
whole store again for every trial it judges."""
from __future__ import annotations

import builtins
import os
import tempfile
import unittest
from contextlib import ExitStack
from pathlib import Path
from types import SimpleNamespace as NS
from unittest.mock import patch

from nullius import workspace
from nullius.judge import questions as Q

from .test_routes import web


def rec(*a, **k):
return {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"},
"usage": {"input_tokens": 100}, "states": {"trial": {"chars": 300}}}


class CalibrationReads(unittest.TestCase):
def setUp(self):
stack = ExitStack()
self.addCleanup(stack.close)
tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-judge-calib-")))
stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces")))
stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True))
stack.enter_context(patch.dict(os.environ, TYPESAFE_API_KEY="ts-fake"))
stack.enter_context(patch.object(web.judge.batch, "ask_states", side_effect=rec))
stack.enter_context(patch.object(web.judge.trajectory, "load", lambda path: NS(error=None)))
stack.enter_context(patch.object(web.judge, "_trial_ids_and_states", lambda trial, traj: {}))
self.client = web.app.test_client()
with self.client.session_transaction() as sess:
sess["csrf"] = "t"

def reads_to_judge_all(self, n):
web.shared.open_workspace(workspace.create(web.shared.WS_HOME, [], f"w{n}"))
job_id = "job"
trials = [NS(trial_name=f"t{i}", has_trajectory=True, trajectory_path="-", started_at=None, outcome="resolved") for i in range(n)]
job = NS(job_id=job_id, trials=trials)
real_open, reads = builtins.open, [0]

def counting_open(path, mode="r", *a, **k):
if "judgements" in str(path) and "r" in mode:
reads[0] += 1
return real_open(path, mode, *a, **k)

with patch.object(web.shared, "_job_for_request", lambda j: job), patch("builtins.open", counting_open):
self.assertEqual(self.client.post(f"/judge/job/{job_id}", data={"csrf": "t"}).status_code, 302)
self.assertEqual(len(web.judge.store.for_job(web.judge.ws_dir(), job_id, {t.trial_name: None for t in trials})), n)
return reads[0]

def test_store_reads_grow_linearly_with_the_trials_judged(self):
counts = {n: self.reads_to_judge_all(n) for n in (50, 100, 200)}
self.assertLess(counts[200], 4 * 200, counts)


if __name__ == "__main__":
unittest.main()
3 changes: 2 additions & 1 deletion app/web/feasibility.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ def judge_feasibility_route(job_id):
todo = summary["todo"]
judge._guard()
done, failed = 0, []
cal = judge.batch_calibration()
for row, side in todo:
try:
judge.judge_control(side, jobs, judge.key()); done += 1
judge.judge_control(side, jobs, judge.key(), calibration=cal); done += 1
except client.JudgeError as e:
failed.append(f"{row.task_base} ({side.control}): {e}")
if e.status in (401, 0):
Expand Down
33 changes: 20 additions & 13 deletions app/web/judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@ def ws_dir():
return ws.dir if ws else None


def batch_calibration() -> dict:
"""The workspace's calibration, read once for a batch and passed to each
call; {} when there is no sample, so no call reads the store again."""
return store.calibration(ws_dir()) or {}


def started_of(trial) -> str | None:
"""A trial's start, as its judgement keeps it: a retry runs an attempt
again under the same name, and an earlier try's record is not its own."""
Expand Down Expand Up @@ -248,16 +254,17 @@ def _trial_ids_and_states(trial, traj) -> dict:
return _states_for(trial, traj, ids)


def judge_trial(job, trial, api_key: str, post=None) -> dict:
def judge_trial(job, trial, api_key: str, post=None, calibration=None) -> dict:
"""Ask every question the trial page shows, one call per state, and keep
the merged record. Returns the record as stored."""
the merged record. Returns the record as stored. `calibration`: a
batch's, read once (`store.calibration`); the workspace's when none."""
if not trial.has_trajectory:
raise client.JudgeError(0, "this trial has no trajectory to judge")
traj = trajectory.load(trial.trajectory_path)
if traj.error:
raise client.JudgeError(0, "the trajectory could not be read: " + traj.error)
return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, _trial_ids_and_states(trial, traj), api_key, post=post,
started=started_of(trial))
calibration=calibration, started=started_of(trial))


def _guard():
Expand Down Expand Up @@ -300,8 +307,8 @@ def judge_job_route(job_id):
judged = store.for_job(d, job_id, started_by_name(job), lane=lane) if d else {}
todo = [t for t in job.trials if t.has_trajectory and t.trial_name not in judged]
_guard()
k = key(lane)
done, failed = run_batch(todo, lambda t: judge_trial(job, t, k), lambda t: t.trial_name)
k, cal = key(lane), batch_calibration()
done, failed = run_batch(todo, lambda t: judge_trial(job, t, k, calibration=cal), lambda t: t.trial_name)
from flask import flash
flash(f"{'LLM judge' if lane == 'llm' else 'Decision model'}: judged {done} of {len(todo)} trial(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "")
return redirect(url_for("job", job_id=job_id) + "#judgement")
Expand Down Expand Up @@ -437,7 +444,7 @@ def compare_summary(view) -> dict | None:
"margin": comparison.TIE_MARGIN, "question": SIDE_QUESTION}


def judge_side(job, trial, api_key: str, post=None) -> dict | None:
def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict | None:
"""Score one side: the questions it still lacks under the current
vocabulary, one call per state, merged over the trial's own record -
the same record the trial page shows. Nothing when it has them all."""
Expand All @@ -453,7 +460,7 @@ def judge_side(job, trial, api_key: str, post=None) -> dict | None:
if traj.error:
raise client.JudgeError(0, "the trajectory could not be read: " + traj.error)
return batch.ask_and_store(d, job.job_id, trial.trial_name, _states_for(trial, traj, ids), api_key, post=post,
started=started_of(trial))
calibration=calibration, started=started_of(trial))


def _sides_todo(view) -> list:
Expand All @@ -477,8 +484,8 @@ def judge_compare_route():
sides = _sides_todo(view)
todo = list(dict.fromkeys(task for task, _, _ in sides))
_guard()
k = key()
done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})")
k, cal = key(), batch_calibration()
done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})")
from flask import flash
flash(f"Judged {done} of {len(sides)} side(s) over {len(todo)} pair(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "")
return redirect(url_for("compare", pick=view.references, cohort=view.cohort, view=compare_web._view()) + "#judgement")
Expand Down Expand Up @@ -532,14 +539,14 @@ def control_states(side, trial) -> dict:
return batch.states_for(Q.group_ids("feasibility"), build)


def judge_control(side, jobs, api_key: str, post=None) -> dict | None:
def judge_control(side, jobs, api_key: str, post=None, calibration=None) -> dict | None:
"""Ask why one control side did not behave, over its own `control` state,
stored as that control trial's record. The verdict is not touched."""
job, trial = _control_trial(side, jobs)
if trial is None:
raise client.JudgeError(0, f"{side.control} trial {side.trial_name} is not in this workspace")
return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, control_states(side, trial), api_key, post=post,
started=started_of(trial))
calibration=calibration, started=started_of(trial))


def triage_summary(report, jobs) -> dict:
Expand Down Expand Up @@ -645,11 +652,11 @@ def task_summary(job) -> dict:
return {"tasks": len(tasks), "judged": judged, "unjudged": len(tasks_todo(job)), "keyed": bool(key())}


def judge_task(job, trial, api_key: str, post=None) -> dict | None:
def judge_task(job, trial, api_key: str, post=None, calibration=None) -> dict | None:
"""Score one task's instruction over the `task` state alone, stored under
the dataset's task slot and the instruction's digest - never under the trial."""
instruction = _task_instruction(trial)
if not instruction:
raise client.JudgeError(0, f"no instruction can be read for {trial.task_base}")
states = batch.states_for(Q.group_ids("task"), lambda kind: judge_state.build(kind, None, instruction=instruction))
return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post)
return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post, calibration=calibration)
4 changes: 2 additions & 2 deletions app/web/judge_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ def judge_tasks_route(job_id):
abort(404)
todo = judge.tasks_todo(job)
judge._guard()
k = judge.key()
done, failed = judge.run_batch(todo, lambda t: judge.judge_task(job, t, k), lambda t: t.task_base)
k, cal = judge.key(), judge.batch_calibration()
done, failed = judge.run_batch(todo, lambda t: judge.judge_task(job, t, k, calibration=cal), lambda t: t.task_base)
flash(f"Judged {done} of {len(todo)} task instruction(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "")
return redirect(url_for("job", job_id=job_id) + "#task-quality")

Expand Down
Loading