From b8dd1c3789155145195283b0d9bb52be67e7384e Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 12:15:50 +0100 Subject: [PATCH] fix(judge): key a task instruction's score by its dataset and its text Task judgements were kept under `task:/`. Two datasets of one folder name, or every job that names no dataset, shared a score, and an instruction edited under the same task name kept the old score, so Judge all for tasks never judged the new text. The slot is now named by the dataset path the job recorded, or its registry name, and a job that names none has a slot of its own. A task's name in the slot carries a digest of the instruction the judge reads, so an edited instruction reads as not judged. The instruction is read once per trajectory file and change, since the job page asks it of every row. Scores kept before this change name no instruction: they stay on disk and read as not judged, so Judge all for tasks asks them again. --- app/nullius/judge/questions.py | 2 +- app/ui_tests/test_task_judgement_slot.py | 67 ++++++++++++++++++++++++ app/web/judge.py | 44 +++++++++++----- 3 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 app/ui_tests/test_task_judgement_slot.py diff --git a/app/nullius/judge/questions.py b/app/nullius/judge/questions.py index 98a3c5ae..e4be0db1 100644 --- a/app/nullius/judge/questions.py +++ b/app/nullius/judge/questions.py @@ -251,7 +251,7 @@ def hygiene_ids() -> list: # --- task-instruction quality: the dataset's own text, no run involved ----------------- # A Score over the `task` state - the instruction alone. Stored once per -# dataset and task (`web.judge.task_key`), not per trial: an ambiguous +# dataset, task and instruction text (`web.judge.task_key`), not per trial: an ambiguous # instruction caps every agent on that task, which is a harness finding. register("task", { "instruction_quality": { diff --git a/app/ui_tests/test_task_judgement_slot.py b/app/ui_tests/test_task_judgement_slot.py new file mode 100644 index 00000000..7232058c --- /dev/null +++ b/app/ui_tests/test_task_judgement_slot.py @@ -0,0 +1,67 @@ +"""A task instruction's score belongs to the instruction it scored: a +changed instruction, or another dataset of the same folder name, is +judged on its own.""" +from __future__ import annotations + +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from types import SimpleNamespace as NS +from unittest.mock import patch + +from nullius import workspace +from nullius.judge import questions as Q, store + +from .test_routes import web + +SCORED = {"questions": Q.VERSION, "judge": {"model": "m"}, + "answers": {"instruction_quality": {"score": 2, "legend": ["bad", "ok", "good"]}}} + + +class TaskSlot(unittest.TestCase): + def setUp(self): + stack = ExitStack() + self.addCleanup(stack.close) + tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-task-slot-"))) + stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces"))) + stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True)) + web.shared.open_workspace(workspace.create(web.shared.WS_HOME, [], "w")) + self.instruction = "Fix the bug in parse.py." + stack.enter_context(patch.object(web.judge, "_task_instruction", lambda t: self.instruction)) + stack.enter_context(patch.object(web.judge.batch, "ask_states", return_value=dict(SCORED))) + + def job(self, job_id, dataset, path=""): + trial = NS(task_base="fix-bug", has_trajectory=True) + return NS(job_id=job_id, dataset=dataset, dataset_path=path, trials=[trial]), trial + + def test_a_changed_instruction_is_judged_again(self): + job, trial = self.job("run-1", "tasks") + web.judge.judge_task(job, trial, None) + self.instruction = "Fix the bug in render.py, and add a test." # the task was edited, same name + self.assertEqual(web.judge.tasks_todo(job), [trial], + "still scored: " + str(web.judge.task_quality_of(job, trial))) + + def test_two_datasets_with_one_folder_name_are_judged_apart(self): + a, ta = self.job("bench-a", "tasks", "/x/bench-a/tasks") # parse.py keeps the basename as the dataset + b, tb = self.job("bench-b", "tasks", "/y/bench-b/tasks") + web.judge.judge_task(a, ta, None) + self.assertIsNone(web.judge.task_quality_of(b, tb), "bench-b shows bench-a's score, slot " + web.judge.task_key(b)) + again, _ = self.job("run-2", "tasks", "/x/bench-a/tasks") + self.assertEqual(web.judge.task_quality_of(again, ta)["score"], 2) # another run of bench-a: judged once for both + + def test_two_jobs_of_no_dataset_are_judged_apart(self): + a, ta = self.job("run-1", "") + b, tb = self.job("run-2", "") + web.judge.judge_task(a, ta, None) + self.assertIsNone(web.judge.task_quality_of(b, tb)) + + def test_a_score_kept_without_its_instruction_is_not_shown(self): + job, trial = self.job("run-1", "tasks", "/x/bench-a/tasks") + store.save(web.judge.ws_dir(), web.judge.task_key(job), "fix-bug", dict(SCORED)) # as kept before the digest + self.assertIsNone(web.judge.task_quality_of(job, trial)) + self.assertEqual(web.judge.tasks_todo(job), [trial]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/judge.py b/app/web/judge.py index 58f0c7fb..b7b06196 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -3,6 +3,8 @@ the recorded evidence, never in its place.""" from __future__ import annotations +import functools +import hashlib import os import threading from concurrent.futures import ThreadPoolExecutor @@ -558,17 +560,25 @@ def triage_summary(report, jobs) -> dict: "question": Q.QUESTIONS["control_failure_cause"]["instructions"] if "control_failure_cause" in Q.QUESTIONS else ""} -# -- task-instruction quality: once per dataset and task --------------------------------- +# -- task-instruction quality: once per dataset, task and instruction text --------------- def task_key(job) -> str: """The store's job slot for a dataset's task judgements: `task:`, - so an instruction is judged once for every run of that dataset.""" - return "task:" + (job.dataset or "unknown") + so an instruction is judged once for every run of that dataset. The + dataset is named by its recorded path, or its registry name, so two + datasets of one folder name keep apart; a job that names none has its own.""" + return "task:" + (job.dataset_path or job.dataset or "job:" + job.job_id) -def task_record(job, task_base: str) -> dict | None: +def task_slot(trial) -> str: + """A task's name in its dataset's slot, with a digest of the instruction + the judge reads: an edited instruction is a task not yet judged.""" + return trial.task_base + "@" + hashlib.sha256(_task_instruction(trial).encode("utf-8")).hexdigest()[:16] + + +def task_record(job, trial) -> dict | None: d = ws_dir() - rec = store.load(d, task_key(job), task_base) if d else None + rec = store.load(d, task_key(job), task_slot(trial)) if d else None return rec if store.is_current(rec) else None @@ -576,7 +586,7 @@ def task_record(job, task_base: str) -> dict | None: def task_quality_of(job, trial) -> dict | None: """The instruction's score for a trial's task: the expectation, the level it rounds to (display only), the judge's confidence; None until judged.""" - rec = task_record(job, trial.task_base) + rec = task_record(job, trial) a = (rec.get("answers") or {}).get("instruction_quality") if rec else None if not a or not isinstance(a.get("score"), (int, float)): return None @@ -602,7 +612,17 @@ def _task_instruction(trial) -> str: """The task's text as the agent received it, from the trial's trajectory.""" if not trial.has_trajectory: return "" - traj = trajectory.load(trial.trajectory_path) + try: + stamp = os.stat(trial.trajectory_path).st_mtime_ns + except OSError: + return "" + return _instruction_at(trial.trajectory_path, stamp) + + +@functools.lru_cache(maxsize=1024) +def _instruction_at(path: str, stamp: int) -> str: + """`_task_instruction` once per trajectory file and change: a job page asks it of every row.""" + traj = trajectory.load(path) return "" if traj.error else judge_state.instruction_of(traj) @@ -613,23 +633,23 @@ def tasks_todo(job) -> list: if t.task_base in seen: continue seen.add(t.task_base) - if task_record(job, t.task_base) is None and t.has_trajectory: + if t.has_trajectory and task_record(job, t) is None: out.append(t) return out def task_summary(job) -> dict: """For the job page: how many of the job's tasks have their instruction judged.""" - tasks = {t.task_base for t in job.trials} - judged = sum(1 for tb in tasks if task_record(job, tb) is not None) + tasks = {t.task_base: t for t in reversed(job.trials)} # each task by its first trial, as tasks_todo reads it + judged = sum(1 for t in tasks.values() if task_record(job, t) is not None) return {"tasks": len(tasks), "judged": judged, "unjudged": len(tasks_todo(job)), "keyed": bool(key())} def judge_task(job, trial, api_key: str, post=None) -> dict | None: """Score one task's instruction over the `task` state alone, stored under - the dataset's task slot - never under the trial.""" + the dataset's task slot and the instruction's digest - never under the trial.""" instruction = _task_instruction(trial) if not instruction: raise client.JudgeError(0, f"no instruction can be read for {trial.task_base}") states = batch.states_for(Q.group_ids("task"), lambda kind: judge_state.build(kind, None, instruction=instruction)) - return batch.ask_and_store(ws_dir(), task_key(job), trial.task_base, states, api_key, post=post) + return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post)