Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/nullius/judge/questions.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ def hygiene_ids() -> list:

# --- task-instruction quality: the dataset's own text, no run involved -----------------
# A Score over the `task` state - the instruction alone. Stored once per
# dataset and task (`web.judge.task_key`), not per trial: an ambiguous
# dataset, task and instruction text (`web.judge.task_key`), not per trial: an ambiguous
# instruction caps every agent on that task, which is a harness finding.
register("task", {
"instruction_quality": {
Expand Down
67 changes: 67 additions & 0 deletions app/ui_tests/test_task_judgement_slot.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
"""A task instruction's score belongs to the instruction it scored: a
changed instruction, or another dataset of the same folder name, is
judged on its own."""
from __future__ import annotations

import tempfile
import unittest
from contextlib import ExitStack
from pathlib import Path
from types import SimpleNamespace as NS
from unittest.mock import patch

from nullius import workspace
from nullius.judge import questions as Q, store

from .test_routes import web

SCORED = {"questions": Q.VERSION, "judge": {"model": "m"},
"answers": {"instruction_quality": {"score": 2, "legend": ["bad", "ok", "good"]}}}


class TaskSlot(unittest.TestCase):
def setUp(self):
stack = ExitStack()
self.addCleanup(stack.close)
tmp = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="nullius-task-slot-")))
stack.enter_context(patch.object(web.shared, "WS_HOME", str(tmp / "workspaces")))
stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}, clear=True))
web.shared.open_workspace(workspace.create(web.shared.WS_HOME, [], "w"))
self.instruction = "Fix the bug in parse.py."
stack.enter_context(patch.object(web.judge, "_task_instruction", lambda t: self.instruction))
stack.enter_context(patch.object(web.judge.batch, "ask_states", return_value=dict(SCORED)))

def job(self, job_id, dataset, path=""):
trial = NS(task_base="fix-bug", has_trajectory=True)
return NS(job_id=job_id, dataset=dataset, dataset_path=path, trials=[trial]), trial

def test_a_changed_instruction_is_judged_again(self):
job, trial = self.job("run-1", "tasks")
web.judge.judge_task(job, trial, None)
self.instruction = "Fix the bug in render.py, and add a test." # the task was edited, same name
self.assertEqual(web.judge.tasks_todo(job), [trial],
"still scored: " + str(web.judge.task_quality_of(job, trial)))

def test_two_datasets_with_one_folder_name_are_judged_apart(self):
a, ta = self.job("bench-a", "tasks", "/x/bench-a/tasks") # parse.py keeps the basename as the dataset
b, tb = self.job("bench-b", "tasks", "/y/bench-b/tasks")
web.judge.judge_task(a, ta, None)
self.assertIsNone(web.judge.task_quality_of(b, tb), "bench-b shows bench-a's score, slot " + web.judge.task_key(b))
again, _ = self.job("run-2", "tasks", "/x/bench-a/tasks")
self.assertEqual(web.judge.task_quality_of(again, ta)["score"], 2) # another run of bench-a: judged once for both

def test_two_jobs_of_no_dataset_are_judged_apart(self):
a, ta = self.job("run-1", "")
b, tb = self.job("run-2", "")
web.judge.judge_task(a, ta, None)
self.assertIsNone(web.judge.task_quality_of(b, tb))

def test_a_score_kept_without_its_instruction_is_not_shown(self):
job, trial = self.job("run-1", "tasks", "/x/bench-a/tasks")
store.save(web.judge.ws_dir(), web.judge.task_key(job), "fix-bug", dict(SCORED)) # as kept before the digest
self.assertIsNone(web.judge.task_quality_of(job, trial))
self.assertEqual(web.judge.tasks_todo(job), [trial])


if __name__ == "__main__":
unittest.main()
44 changes: 32 additions & 12 deletions app/web/judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
the recorded evidence, never in its place."""
from __future__ import annotations

import functools
import hashlib
import os
import threading
from concurrent.futures import ThreadPoolExecutor
Expand Down Expand Up @@ -558,25 +560,33 @@ def triage_summary(report, jobs) -> dict:
"question": Q.QUESTIONS["control_failure_cause"]["instructions"] if "control_failure_cause" in Q.QUESTIONS else ""}


# -- task-instruction quality: once per dataset and task ---------------------------------
# -- task-instruction quality: once per dataset, task and instruction text ---------------

def task_key(job) -> str:
"""The store's job slot for a dataset's task judgements: `task:<dataset>`,
so an instruction is judged once for every run of that dataset."""
return "task:" + (job.dataset or "unknown")
so an instruction is judged once for every run of that dataset. The
dataset is named by its recorded path, or its registry name, so two
datasets of one folder name keep apart; a job that names none has its own."""
return "task:" + (job.dataset_path or job.dataset or "job:" + job.job_id)


def task_record(job, task_base: str) -> dict | None:
def task_slot(trial) -> str:
"""A task's name in its dataset's slot, with a digest of the instruction
the judge reads: an edited instruction is a task not yet judged."""
return trial.task_base + "@" + hashlib.sha256(_task_instruction(trial).encode("utf-8")).hexdigest()[:16]


def task_record(job, trial) -> dict | None:
d = ws_dir()
rec = store.load(d, task_key(job), task_base) if d else None
rec = store.load(d, task_key(job), task_slot(trial)) if d else None
return rec if store.is_current(rec) else None


@app.template_global()
def task_quality_of(job, trial) -> dict | None:
"""The instruction's score for a trial's task: the expectation, the level
it rounds to (display only), the judge's confidence; None until judged."""
rec = task_record(job, trial.task_base)
rec = task_record(job, trial)
a = (rec.get("answers") or {}).get("instruction_quality") if rec else None
if not a or not isinstance(a.get("score"), (int, float)):
return None
Expand All @@ -602,7 +612,17 @@ def _task_instruction(trial) -> str:
"""The task's text as the agent received it, from the trial's trajectory."""
if not trial.has_trajectory:
return ""
traj = trajectory.load(trial.trajectory_path)
try:
stamp = os.stat(trial.trajectory_path).st_mtime_ns
except OSError:
return ""
return _instruction_at(trial.trajectory_path, stamp)


@functools.lru_cache(maxsize=1024)
def _instruction_at(path: str, stamp: int) -> str:
"""`_task_instruction` once per trajectory file and change: a job page asks it of every row."""
traj = trajectory.load(path)
return "" if traj.error else judge_state.instruction_of(traj)


Expand All @@ -613,23 +633,23 @@ def tasks_todo(job) -> list:
if t.task_base in seen:
continue
seen.add(t.task_base)
if task_record(job, t.task_base) is None and t.has_trajectory:
if t.has_trajectory and task_record(job, t) is None:
out.append(t)
return out


def task_summary(job) -> dict:
"""For the job page: how many of the job's tasks have their instruction judged."""
tasks = {t.task_base for t in job.trials}
judged = sum(1 for tb in tasks if task_record(job, tb) is not None)
tasks = {t.task_base: t for t in reversed(job.trials)} # each task by its first trial, as tasks_todo reads it
judged = sum(1 for t in tasks.values() if task_record(job, t) is not None)
return {"tasks": len(tasks), "judged": judged, "unjudged": len(tasks_todo(job)), "keyed": bool(key())}


def judge_task(job, trial, api_key: str, post=None) -> dict | None:
"""Score one task's instruction over the `task` state alone, stored under
the dataset's task slot - never under the trial."""
the dataset's task slot and the instruction's digest - never under the trial."""
instruction = _task_instruction(trial)
if not instruction:
raise client.JudgeError(0, f"no instruction can be read for {trial.task_base}")
states = batch.states_for(Q.group_ids("task"), lambda kind: judge_state.build(kind, None, instruction=instruction))
return batch.ask_and_store(ws_dir(), task_key(job), trial.task_base, states, api_key, post=post)
return batch.ask_and_store(ws_dir(), task_key(job), task_slot(trial), states, api_key, post=post)
Loading