From f935a3f7646dbbd55335759a6b4d87c01518a708 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 12:20:30 +0100 Subject: [PATCH] fix(judge): keep a job's judgements in the workspace that holds it A job opened from another workspace (`?workspace=`) and a Compare column of another workspace read and wrote their judgements in the open workspace's store: the page did not show the judgement kept with the job, and a Judge click wrote one where deleting the open workspace would lose it. The judge's store directory now comes from the workspace that holds the job on screen: the request's `?workspace=` (the same lookup the pages use for the job), else the open workspace. Compare's sides use each column's own workspace. The Judge, Judge all, task and feasibility forms, and the reconcile and feasibility links, keep `?workspace=` so the post finds the same job. Judgements stay inside workspaces. --- app/ui_tests/test_every_route.py | 2 +- app/ui_tests/test_judge_other_workspace.py | 72 ++++++++++++++++++++++ app/web/judge.py | 40 ++++++------ app/web/reconcile.py | 7 +-- app/web/shared.py | 10 ++- 5 files changed, 103 insertions(+), 28 deletions(-) create mode 100644 app/ui_tests/test_judge_other_workspace.py diff --git a/app/ui_tests/test_every_route.py b/app/ui_tests/test_every_route.py index 905adab8..b6aec388 100644 --- a/app/ui_tests/test_every_route.py +++ b/app/ui_tests/test_every_route.py @@ -244,7 +244,7 @@ def test_the_batch_judge_routes_judge_on_a_post(self): with patch.object(judge, "judge_trial", stub), patch.object(judge, "judge_side", stub), \ patch.object(judge, "judge_task", stub), patch.object(judge, "judge_control", stub), \ patch.object(judge, "tasks_todo", lambda j: list(j.trials)), \ - patch.object(judge, "_sides_todo", lambda view: [(t.task_base, job, t) for t in job.trials]), \ + patch.object(judge, "_sides_todo", lambda view: [(t.task_base, job, t, None) for t in job.trials]), \ patch.dict(os.environ, TYPESAFE_API_KEY="ts-fake"): with self.client.session_transaction() as sess: sess["csrf"] = "route-token" diff --git a/app/ui_tests/test_judge_other_workspace.py b/app/ui_tests/test_judge_other_workspace.py new file mode 100644 index 00000000..558349fc --- /dev/null +++ b/app/ui_tests/test_judge_other_workspace.py @@ -0,0 +1,72 @@ +"""A job of another workspace, opened with ``?workspace=``, shows and keeps its +judgements in that workspace's store, not the open one's.""" +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import patch + +from flask import url_for + +from nullius import workspace +from nullius.judge import questions as Q, store + +from .test_routes import web + +JOB = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "harbor-jobs" / "liveboard-oracle-x4" +TRIAL = "hello-world__mAXMKXR" + + +class JudgementsOfAnotherWorkspace(unittest.TestCase): + def setUp(self): + stack = ExitStack() + self.addCleanup(stack.close) + tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-judge-ws-"))) + shutil.copytree(JOB, tmp / "a" / "job-x") + shutil.copytree(JOB, tmp / "b" / "job-y") + stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces"))) + stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True)) + self.a = workspace.create(web.shared.WS_HOME, [str(tmp / "a")], "A") + self.b = workspace.create(web.shared.WS_HOME, [str(tmp / "b")], "B") + web.shared.open_workspace(self.a) + self.url = f"/trial/job-y/{TRIAL}?workspace={self.b.name}" + + def test_the_trial_page_reads_the_judgement_kept_in_its_own_workspace(self): + store.save(self.b.dir, "job-y", TRIAL, {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"}, + "asked_at": "2026-11-01T00:00:00Z"}) + seen = [] + real = web.traces._judgement_panel + with patch.object(web.traces, "_judgement_panel", side_effect=lambda *a: seen.append(real(*a)) or seen[-1]): + self.assertEqual(web.app.test_client().get(self.url).status_code, 200) + self.assertTrue(seen[0]["judged"], "B's judgement of B's trial is not shown") + + def test_judging_that_trial_writes_into_its_own_workspace(self): + rec = {"questions": Q.VERSION, "answers": {"q": {"choice": "x"}}, "judge": {"model": "m"}} + with web.app.test_request_context(self.url): + job, trial = web.traces._trial_evidence("job-y", TRIAL) + with patch.object(web.judge.batch, "ask_states", return_value=dict(rec)): + web.judge.batch.ask_and_store(web.judge.ws_dir(), job.job_id, trial.trial_name, {}, None) + self.assertNotIn("judgements", os.listdir(self.a.dir), "a judgement of B's trial was written into A") + self.assertIsNotNone(store.load(self.b.dir, "job-y", TRIAL)) + + def test_the_judge_forms_post_back_to_that_workspace(self): + with web.app.test_request_context(self.url): + for endpoint, args in (("judge_trial_route", {"trial_name": TRIAL}), ("judge_job_route", {}), ("judge_tasks_route", {})): + self.assertIn(f"workspace={self.b.name}", url_for(endpoint, job_id="job-y", **args)) + + def test_each_compare_column_reads_its_own_workspace(self): + with web.app.test_request_context(f"/compare?pick={self.a.name}:job-x&pick={self.b.name}:job-y"): + view = web.compare._comparison_requested() + self.assertEqual([web.judge.ws_dir(s.workspace) for s in view.selected], [self.a.dir, self.b.dir]) + + def test_a_job_of_the_open_workspace_keeps_its_store(self): + with web.app.test_request_context(f"/trial/job-x/{TRIAL}"): + self.assertEqual(web.judge.ws_dir(), self.a.dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/judge.py b/app/web/judge.py index 7edcae59..9c05b2dc 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -59,8 +59,11 @@ def one(item): return done, failed -def ws_dir(): - ws = shared.STATE.get("workspace") +def ws_dir(name=None): + """The judgement store of the workspace that holds the job on screen + (`shared._record_workspace`): a job opened from another workspace, or a + Compare column (`name`), keeps its judgements in its own.""" + ws = shared._record_workspace(name) return ws.dir if ws else None @@ -387,11 +390,10 @@ def side_ids(trial) -> list: return own + [q for q in Q.for_trial(trial.outcome == "resolved", trial.outcome) if q not in own] -def side_of(job, trial) -> dict | None: +def side_of(job, trial, d) -> dict | None: """One side's score as the compare page shows it, from the trial's own - current record: the expectation, the level it rounds to, the judge's + current record in its column's store `d`: the expectation, the level it rounds to, the judge's confidence, the provenance. None until scored under the current vocabulary.""" - d = ws_dir() rec = store.load(d, job.job_id, trial.trial_name, started=started_of(trial)) if d else None if not rec or not store.is_current(rec): return None @@ -406,8 +408,8 @@ def side_of(job, trial) -> dict | None: def pair_records(view) -> dict: """{task: record} for the pairs judged under v1 - the pair state, the - model's own pick - under job A in the open workspace. Read, never written.""" - d = ws_dir() + model's own pick - under job A in its column's workspace. Read, never written.""" + d = ws_dir(view.selected[0].workspace) if view.selected else None if d is None or len(view.jobs) != 2: return {} out = {} @@ -429,8 +431,9 @@ def compare_summary(view) -> dict | None: pairs = pairs_of(view) legacy = pair_records(view) rows, judged, earlier = {}, 0, 0 + da, db = (ws_dir(s.workspace) for s in view.selected) for task, a, b in pairs: - sa, sb = side_of(view.jobs[0], a), side_of(view.jobs[1], b) + sa, sb = side_of(view.jobs[0], a, da), side_of(view.jobs[1], b, db) if sa or sb: verdict = comparison.better_of(sa and sa["score"], sb and sb["score"]) rows[task] = {"a": sa, "b": sb, "verdict": verdict, "earlier": False} @@ -444,13 +447,13 @@ def compare_summary(view) -> dict | None: "margin": comparison.TIE_MARGIN, "question": SIDE_QUESTION} -def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict | None: +def judge_side(job, trial, d, api_key: str, post=None, calibration=None) -> dict | None: """Score one side: the questions it still lacks under the current - vocabulary, one call per state, merged over the trial's own record - - the same record the trial page shows. Nothing when it has them all.""" + vocabulary, one call per state, merged over the trial's own record in + its column's store `d` - the same record the trial page shows. Nothing + when it has them all.""" if not trial.has_trajectory: raise client.JudgeError(0, "this trial has no trajectory to judge") - d = ws_dir() rec = store.load(d, job.job_id, trial.trial_name, started=started_of(trial)) have = (rec.get("answers") or {}) if store.is_current(rec) else {} ids = [q for q in side_ids(trial) if q not in have] @@ -464,12 +467,13 @@ def judge_side(job, trial, api_key: str, post=None, calibration=None) -> dict | def _sides_todo(view) -> list: - """(job, trial) for every side of an unjudged pair that lacks its score.""" + """(task, job, trial, its column's store) for every side of an unjudged pair that lacks its score.""" out = [] + da, db = (ws_dir(s.workspace) for s in view.selected) for task, a, b in pairs_of(view): - for job, t in ((view.jobs[0], a), (view.jobs[1], b)): - if side_of(job, t) is None: - out.append((task, job, t)) + for job, t, d in ((view.jobs[0], a, da), (view.jobs[1], b, db)): + if side_of(job, t, d) is None: + out.append((task, job, t, d)) return out @@ -482,10 +486,10 @@ def judge_compare_route(): if len(view.jobs) != 2: abort(400, description="A judged comparison needs exactly two runs.") sides = _sides_todo(view) - todo = list(dict.fromkeys(task for task, _, _ in sides)) + todo = list(dict.fromkeys(s[0] for s in sides)) _guard() k, cal = key(), batch_calibration() - done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})") + done, failed = run_batch(sides, lambda s: judge_side(s[1], s[2], s[3], k, calibration=cal), lambda s: f"{s[0]} ({s[1].agent_label or s[1].job_id})") from flask import flash flash(f"Judged {done} of {len(sides)} side(s) over {len(todo)} pair(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "") return redirect(url_for("compare", pick=view.references, cohort=view.cohort, view=compare_web._view()) + "#judgement") diff --git a/app/web/reconcile.py b/app/web/reconcile.py index e288d161..6d0d0f4b 100644 --- a/app/web/reconcile.py +++ b/app/web/reconcile.py @@ -14,11 +14,6 @@ from . import judge, shared -def ws_dir(): - ws = shared.STATE.get("workspace") - return ws.dir if ws else None - - def threshold_requested() -> float: """`?threshold=` as a number in 0..1, else the module's default.""" raw = request.args.get("threshold") @@ -30,7 +25,7 @@ def threshold_requested() -> float: def view_of(job, threshold: float) -> dict: - d = ws_dir() + d = judge.ws_dir() records = store.for_job(d, job.job_id, judge.started_by_name(job)) if d else {} return reconcile.reconcile(job, records, threshold) diff --git a/app/web/shared.py b/app/web/shared.py index a0575d98..337f3007 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -419,8 +419,11 @@ def _jobs_for_workspace(name=None): return jobs -def _record_workspace(): - name = request.args.get("workspace") if has_request_context() else None +def _record_workspace(name=None): + """The workspace that holds the record on screen: `name` ("" the open + one), else this request's `?workspace=`, else the open one.""" + if name is None: + name = request.args.get("workspace") if has_request_context() else None if not name: return STATE["workspace"] ws = workspace.load(WS_HOME, name) @@ -437,7 +440,8 @@ def _jobs_for_request(): def _preserve_record_workspace(endpoint, values): """Carry evidence provenance through links, forms, and lazy detail requests.""" record_endpoints = {"job", "trial", "save_trace", "artifact_page", "conversation_data", - "trace_span_data"} + "trace_span_data", "reconcile_page", "feasibility_page", "judge_trial_route", + "judge_job_route", "judge_tasks_route", "judge_feasibility_route"} if has_request_context() and endpoint in record_endpoints and "workspace" not in values: name = request.args.get("workspace") if name: