Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/ui_tests/test_every_route.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ def test_the_batch_judge_routes_judge_on_a_post(self):
with patch.object(judge, "judge_trial", stub), patch.object(judge, "judge_side", stub), \
patch.object(judge, "judge_task", stub), patch.object(judge, "judge_control", stub), \
patch.object(judge, "tasks_todo", lambda j: list(j.trials)), \
patch.object(judge, "_sides_todo", lambda view: [(t.task_base, job, t) for t in job.trials]), \
patch.object(judge, "_sides_todo", lambda view: [(t.task_base, job, t, None) for t in job.trials]), \
patch.dict(os.environ, TYPESAFE_API_KEY="ts-fake"):
with self.client.session_transaction() as sess:
sess["csrf"] = "route-token"
Expand Down
72 changes: 72 additions & 0 deletions app/ui_tests/test_judge_other_workspace.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""A job of another workspace, opened with ``?workspace=``, shows and keeps its
judgements in that workspace's store, not the open one's."""
from __future__ import annotations

import os
import shutil
import tempfile
import unittest
from contextlib import ExitStack
from pathlib import Path
from unittest.mock import patch

from flask import url_for

from nullius import workspace
from nullius.judge import questions as Q, store

from .test_routes import web

JOB = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "harbor-jobs" / "liveboard-oracle-x4"
TRIAL = "hello-world__mAXMKXR"


class JudgementsOfAnotherWorkspace(unittest.TestCase):
def setUp(self):
stack = ExitStack()
self.addCleanup(stack.close)
tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-judge-ws-")))
shutil.copytree(JOB, tmp / "a" / "job-x")
shutil.copytree(JOB, tmp / "b" / "job-y")
stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces")))
stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True))
self.a = workspace.create(web.shared.WS_HOME, [str(tmp / "a")], "A")
self.b = workspace.create(web.shared.WS_HOME, [str(tmp / "b")], "B")
web.shared.open_workspace(self.a)
self.url = f"/trial/job-y/{TRIAL}?workspace={self.b.name}"

def test_the_trial_page_reads_the_judgement_kept_in_its_own_workspace(self):
store.save(self.b.dir, "job-y", TRIAL, {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"},
"asked_at": "2026-11-01T00:00:00Z"})
seen = []
real = web.traces._judgement_panel
with patch.object(web.traces, "_judgement_panel", side_effect=lambda *a: seen.append(real(*a)) or seen[-1]):
self.assertEqual(web.app.test_client().get(self.url).status_code, 200)
self.assertTrue(seen[0]["judged"], "B's judgement of B's trial is not shown")

def test_judging_that_trial_writes_into_its_own_workspace(self):
rec = {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"}}
with web.app.test_request_context(self.url):
job, trial = web.traces._trial_evidence("job-y", TRIAL)
with patch.object(web.judge.batch, "ask_states", return_value=dict(rec)):
web.judge.batch.ask_and_store(web.judge.ws_dir(), job.job_id, trial.trial_name, {}, None)
self.assertNotIn("judgements", os.listdir(self.a.dir), "a judgement of B's trial was written into A")
self.assertIsNotNone(store.load(self.b.dir, "job-y", TRIAL))

def test_the_judge_forms_post_back_to_that_workspace(self):
with web.app.test_request_context(self.url):
for endpoint, args in (("judge_trial_route", {"trial_name": TRIAL}), ("judge_job_route", {}), ("judge_tasks_route", {})):
self.assertIn(f"workspace={self.b.name}", url_for(endpoint, job_id="job-y", **args))

def test_each_compare_column_reads_its_own_workspace(self):
with web.app.test_request_context(f"/compare?pick={self.a.name}:job-x&pick={self.b.name}:job-y"):
view = web.compare._comparison_requested()
self.assertEqual([web.judge.ws_dir(s.workspace) for s in view.selected], [self.a.dir, self.b.dir])

def test_a_job_of_the_open_workspace_keeps_its_store(self):
with web.app.test_request_context(f"/trial/job-x/{TRIAL}"):
self.assertEqual(web.judge.ws_dir(), self.a.dir)


if __name__ == "__main__":
unittest.main()
40 changes: 22 additions & 18 deletions app/web/judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,11 @@ def one(item):
return done, failed


def ws_dir():
ws = shared.STATE.get("workspace")
def ws_dir(name=None):
"""The judgement store of the workspace that holds the job on screen
(`shared._record_workspace`): a job opened from another workspace, or a
Compare column (`name`), keeps its judgements in its own."""
ws = shared._record_workspace(name)
return ws.dir if ws else None


Expand Down Expand Up @@ -387,11 +390,10 @@ def side_ids(trial) -> list:
return own + [q for q in Q.for_trial(trial.outcome == "resolved", trial.outcome) if q not in own]


def side_of(job, trial) -> dict | None:
def side_of(job, trial, d) -> dict | None:
"""One side's score as the compare page shows it, from the trial's own
current record: the expectation, the level it rounds to, the judge's
current record in its column's store `d`: the expectation, the level it rounds to, the judge's
confidence, the provenance. None until scored under the current vocabulary."""
d = ws_dir()
rec = store.load(d, job.job_id, trial.trial_name, started=started_of(trial)) if d else None
if not rec or not store.is_current(rec):
return None
Expand All @@ -406,8 +408,8 @@ def side_of(job, trial) -> dict | None:

def pair_records(view) -> dict:
"""{task: record} for the pairs judged under v1 - the pair state, the
model's own pick - under job A in the open workspace. Read, never written."""
d = ws_dir()
model's own pick - under job A in its column's workspace. Read, never written."""
d = ws_dir(view.selected[0].workspace) if view.selected else None
if d is None or len(view.jobs) != 2:
return {}
out = {}
Expand All @@ -429,8 +431,9 @@ def compare_summary(view) -> dict | None:
pairs = pairs_of(view)
legacy = pair_records(view)
rows, judged, earlier = {}, 0, 0
da, db = (ws_dir(s.workspace) for s in view.selected)
for task, a, b in pairs:
sa, sb = side_of(view.jobs[0], a), side_of(view.jobs[1], b)
sa, sb = side_of(view.jobs[0], a, da), side_of(view.jobs[1], b, db)
if sa or sb:
verdict = comparison.better_of(sa and sa["score"], sb and sb["score"])
rows[task] = {"a": sa, "b": sb, "verdict": verdict, "earlier": False}
Expand All @@ -444,13 +447,13 @@ def compare_summary(view) -> dict | None:
"margin": comparison.TIE_MARGIN, "question": SIDE_QUESTION}


def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict | None:
def judge_side(job, trial, d, api_key: str, post=None, calibration=None) -> dict | None:
"""Score one side: the questions it still lacks under the current
vocabulary, one call per state, merged over the trial's own record -
the same record the trial page shows. Nothing when it has them all."""
vocabulary, one call per state, merged over the trial's own record in
its column's store `d` - the same record the trial page shows. Nothing
when it has them all."""
if not trial.has_trajectory:
raise client.JudgeError(0, "this trial has no trajectory to judge")
d = ws_dir()
rec = store.load(d, job.job_id, trial.trial_name, started=started_of(trial))
have = (rec.get("answers") or {}) if store.is_current(rec) else {}
ids = [q for q in side_ids(trial) if q not in have]
Expand All @@ -464,12 +467,13 @@ def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict |


def _sides_todo(view) -> list:
"""(job, trial) for every side of an unjudged pair that lacks its score."""
"""(task, job, trial, its column's store) for every side of an unjudged pair that lacks its score."""
out = []
da, db = (ws_dir(s.workspace) for s in view.selected)
for task, a, b in pairs_of(view):
for job, t in ((view.jobs[0], a), (view.jobs[1], b)):
if side_of(job, t) is None:
out.append((task, job, t))
for job, t, d in ((view.jobs[0], a, da), (view.jobs[1], b, db)):
if side_of(job, t, d) is None:
out.append((task, job, t, d))
return out


Expand All @@ -482,10 +486,10 @@ def judge_compare_route():
if len(view.jobs) != 2:
abort(400, description="A judged comparison needs exactly two runs.")
sides = _sides_todo(view)
todo = list(dict.fromkeys(task for task, _, _ in sides))
todo = list(dict.fromkeys(s[0] for s in sides))
_guard()
k, cal = key(), batch_calibration()
done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})")
done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], s[3], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})")
from flask import flash
flash(f"Judged {done} of {len(sides)} side(s) over {len(todo)} pair(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "")
return redirect(url_for("compare", pick=view.references, cohort=view.cohort, view=compare_web._view()) + "#judgement")
Expand Down
7 changes: 1 addition & 6 deletions app/web/reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@
from . import judge, shared


def ws_dir():
ws = shared.STATE.get("workspace")
return ws.dir if ws else None


def threshold_requested() -> float:
"""`?threshold=` as a number in 0..1, else the module's default."""
raw = request.args.get("threshold")
Expand All @@ -30,7 +25,7 @@ def threshold_requested() -> float:


def view_of(job, threshold: float) -> dict:
d = ws_dir()
d = judge.ws_dir()
records = store.for_job(d, job.job_id, judge.started_by_name(job)) if d else {}
return reconcile.reconcile(job, records, threshold)

Expand Down
10 changes: 7 additions & 3 deletions app/web/shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -419,8 +419,11 @@ def _jobs_for_workspace(name=None):
return jobs


def _record_workspace():
name = request.args.get("workspace") if has_request_context() else None
def _record_workspace(name=None):
"""The workspace that holds the record on screen: `name` ("" the open
one), else this request's `?workspace=`, else the open one."""
if name is None:
name = request.args.get("workspace") if has_request_context() else None
if not name:
return STATE["workspace"]
ws = workspace.load(WS_HOME, name)
Expand All @@ -437,7 +440,8 @@ def _jobs_for_request():
def _preserve_record_workspace(endpoint, values):
"""Carry evidence provenance through links, forms, and lazy detail requests."""
record_endpoints = {"job", "trial", "save_trace", "artifact_page", "conversation_data",
"trace_span_data"}
"trace_span_data", "reconcile_page", "feasibility_page", "judge_trial_route",
"judge_job_route", "judge_tasks_route", "judge_feasibility_route"}
if has_request_context() and endpoint in record_endpoints and "workspace" not in values:
name = request.args.get("workspace")
if name:
Expand Down
Loading