Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions app/nullius/judge/lanes.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
_LINE_ORDER = {AGREE_LANE: 0, DISAGREE_LANE: 1, GUTTER: 2}


def board(job, records: dict, threshold: float = R.THRESHOLD, key=None) -> dict | None:
def board(job, records: dict, threshold: float = R.THRESHOLD) -> dict | None:
"""The lanes for one job, or None for a control (nothing to reconcile).

Returns ``rows`` (one per outcome present, in the classifier's order,
Expand All @@ -48,8 +48,7 @@ def board(job, records: dict, threshold: float = R.THRESHOLD, key=None) -> dict
trial) and which lanes any line reaches."""
if not R.applies(job):
return None
key = key or (lambda name: name)
rows = {t.trial_name: R.row_of(t, records.get(key(t.trial_name)), threshold) for t in job.trials}
rows = {t.trial_name: R.row_of(t, records.get(t.trial_name), threshold) for t in job.trials}
status = {}
out = []
for label in classify.ORDER:
Expand Down
9 changes: 3 additions & 6 deletions app/nullius/judge/reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,14 +154,11 @@ def _rank(row: dict) -> tuple:
return (-(row["confidence"] or 0.0), -(row["p"] or 0.0), row["trial_name"])


def reconcile(job, records: dict, threshold: float = THRESHOLD, key=None) -> dict:
def reconcile(job, records: dict, threshold: float = THRESHOLD) -> dict:
"""The job's reconciliation: rows by status (each list ranked strongest
first), and the counts the page states. `records` is the store's
{stored name: record}; `key` maps a trial name to its stored name (the
store's own sanitiser), identity when not given. Reads the job; writes
nothing."""
key = key or (lambda name: name)
rows = [row_of(t, records.get(key(t.trial_name)), threshold) for t in job.trials]
{trial name: record}. Reads the job; writes nothing."""
rows = [row_of(t, records.get(t.trial_name), threshold) for t in job.trials]
by_status = {s: sorted((r for r in rows if r["status"] == s), key=_rank) for s in STATUSES}
judged = sum(1 for r in rows if r["choice"] is not None)
asked = sum(1 for t in job.trials if t.outcome not in (classify.RESOLVED, classify.SCORED))
Expand Down
82 changes: 47 additions & 35 deletions app/nullius/judge/store.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Where a judgement lives: the workspace's own store, beside the parsed
jobs - ``<workspace>/judgements/<job_id>/<trial_name>.json`` for System
jobs - ``<workspace>/judgements/<job_id>/<trial_name>.json`` (each name
spelled by `key`) for System
One's, ``<workspace>/judgements-llm/...`` for the LLM judge's (a trial has
both, apart, ``lane``) - keyed by question version inside. Never under the job's ``agent/`` directory: the
run tree is Harbor's, and a re-parse of the job does not touch this.
Expand All @@ -11,6 +12,7 @@
carries - and a write always lands under the version the record names."""
from __future__ import annotations

import hashlib
import json
import os
import re
Expand Down Expand Up @@ -39,8 +41,36 @@ def _root(ws_dir: str, lane: str) -> str:
return os.path.join(ws_dir, ROOTS.get(lane, "judgements"))


def _path(ws_dir: str, job_id: str, trial_name: str, lane: str = "decision") -> str:
return os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id), _SAFE.sub("_", trial_name) + ".json")
def key(name: str) -> str:
"""A job id's or trial name's file name: the name itself when every
character is safe, else its safe spelling and a digest of the name, so
two names never share a file (no safe spelling holds a ``~``)."""
safe = _SAFE.sub("_", name)
return safe if safe == name else f"{safe}~{hashlib.sha256(name.encode('utf-8')).hexdigest()[:10]}"


def _names(name: str) -> tuple:
"""`key`, then the safe spelling alone that a store written before `key` used, still read."""
return tuple(dict.fromkeys((key(name), _SAFE.sub("_", name))))


def _paths(ws_dir: str, job_id: str, trial_name: str, lane: str = "decision") -> list:
"""Where a judgement is written, then where an older store may hold it."""
root = _root(ws_dir, lane)
return [os.path.join(root, j, t + ".json") for j in _names(job_id) for t in _names(trial_name)]


def _read(paths) -> dict:
"""The first judgement file of `paths` there is, read; {} when there is none or it cannot be read."""
for path in paths:
try:
with open(path, encoding="utf-8") as fh:
return json.load(fh) or {}
except FileNotFoundError:
continue
except (OSError, ValueError):
return {}
return {}


def _versions(version: Optional[str]) -> tuple:
Expand Down Expand Up @@ -68,11 +98,7 @@ def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = Non
"""The record under `version`; with none named, the current version or
the newest earlier one that has a record. `started`: the trial's own
start, so a record of an earlier try under its name is none of its."""
try:
with open(_path(ws_dir, job_id, trial_name, lane), encoding="utf-8") as fh:
return _pick(json.load(fh) or {}, version, started)
except (OSError, ValueError):
return None
return _pick(_read(_paths(ws_dir, job_id, trial_name, lane)), version, started)


def is_current(record: Optional[dict]) -> bool:
Expand All @@ -86,14 +112,11 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "d
record's answers and hygiene readings merge over the version's earlier
ones, so judging one question later does not drop the others. Saves to
one file take turns; each writes through a temporary file of its own."""
path = _path(ws_dir, job_id, trial_name, lane)
paths = _paths(ws_dir, job_id, trial_name, lane)
path = paths[0]
os.makedirs(os.path.dirname(path), exist_ok=True)
with _lock(path):
try:
with open(path, encoding="utf-8") as fh:
data = json.load(fh) or {}
except (OSError, ValueError):
data = {}
data = _read(paths)
version = record.get("questions", Q.VERSION)
earlier = data.get(version) or {}
if not _of(earlier, started):
Expand All @@ -114,33 +137,22 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "d
return merged


def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "decision",
started: Optional[dict] = None) -> dict:
"""Every judged trial of a job: {trial_name: record}, each the current
version's record or the newest earlier one, as `load` picks it.
`started`: {trial_name: its start}, as `load` takes it."""
d = os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id))
def for_job(ws_dir: str, job_id: str, started: dict, version: Optional[str] = None, lane: str = "decision") -> dict:
"""The judged trials of a job: {trial_name: record} for each trial of
`started` - {trial_name: its start}, every trial of the job - that has
one, as `load` picks it. Nothing else under the job's name is a trial of
it: a v1 pair record is not."""
out = {}
try:
names = os.listdir(d)
except OSError:
return out
for n in names:
if not n.endswith(".json"):
continue
try:
with open(os.path.join(d, n), encoding="utf-8") as fh:
rec = _pick(json.load(fh) or {}, version, (started or {}).get(n[:-5]))
except (OSError, ValueError):
continue
for name, start in started.items():
rec = load(ws_dir, job_id, name, version, lane, start)
if rec:
out[n[:-5]] = rec
out[name] = rec
return out


def job_dirs(ws_dir: str, job_ids) -> list:
"""The folders that hold these jobs' judgements, in every lane."""
return [d for lane in ROOTS for j in job_ids if os.path.isdir(d := os.path.join(_root(ws_dir, lane), _SAFE.sub("_", j)))]
return [d for lane in ROOTS for j in job_ids for n in _names(j) if os.path.isdir(d := os.path.join(_root(ws_dir, lane), n))]


def count(ws_dir: str) -> int:
Expand All @@ -154,7 +166,7 @@ def all_records(ws_dir: str, job_id: Optional[str] = None, lane: str = "decision
version, record). One job's when `job_id` is given."""
root = _root(ws_dir, lane)
try:
jobs = [_SAFE.sub("_", job_id)] if job_id else sorted(os.listdir(root))
jobs = list(_names(job_id)) if job_id else sorted(os.listdir(root))
except OSError:
return
for j in jobs:
Expand Down
13 changes: 7 additions & 6 deletions app/tests/test_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,15 +193,16 @@ def test_save_load_merge_and_survive_a_reparse(self):
rec = {"questions": "judgement.questions.v1", "judge": {"provider": "typesafe.ai", "model": "jev-latest"}, "asked_at": "2026-09-17T09:00:00Z",
"state_chars": 10, "truncated": False, "answers": {"task_completed": {"noul": 0.9}}, "usage": {"input_tokens": 1, "output_tokens": 1}}
store.save(self.ws.dir, "job-1", "task__abc/def", rec)
path = Path(self.ws.dir) / "judgements" / "job-1" / "task__abc_def.json"
path = Path(self.ws.dir) / "judgements" / "job-1" / (store.key("task__abc/def") + ".json")
self.assertTrue(path.exists()); self.assertFalse((self.tmp / "jobs").exists()) # under the workspace, never the job tree
self.assertEqual(store.load(self.ws.dir, "job-1", "task__abc/def")["answers"], {"task_completed": {"noul": 0.9}})
later = dict(rec, asked_at="2026-09-17T10:00:00Z", answers={"transcript_quality": {"score": 2.0, "legend": ["a", "b", "c"], "probabilities": {}, "confidence": 0.9}})
store.save(self.ws.dir, "job-1", "task__abc/def", later)
got = store.load(self.ws.dir, "job-1", "task__abc/def")
self.assertEqual(sorted(got["answers"]), ["task_completed", "transcript_quality"]); self.assertEqual(got["asked_at"], "2026-09-17T10:00:00Z") # answers merge per version
self.assertIsNone(store.load(self.ws.dir, "job-1", "task__abc/def", version="judgement.questions.v3"))
self.assertEqual(list(store.for_job(self.ws.dir, "job-1")), ["task__abc_def"]); self.assertEqual(store.for_job(self.ws.dir, "job-2"), {})
trials = {"task__abc/def": None, "other": None}
self.assertEqual(list(store.for_job(self.ws.dir, "job-1", trials)), ["task__abc/def"]); self.assertEqual(store.for_job(self.ws.dir, "job-2", trials), {})
(Path(self.ws.jobs_dir)).mkdir(parents=True, exist_ok=True); (Path(self.ws.jobs_dir) / "x.json").write_text("{}")
workspace.clear_cache(self.ws) # a re-parse of the job
self.assertEqual(store.load(self.ws.dir, "job-1", "task__abc/def")["answers"]["task_completed"], {"noul": 0.9})
Expand Down Expand Up @@ -233,15 +234,15 @@ def test_v1_and_v2_coexist_and_the_read_prefers_the_current(self):
store.save(d, "job", "t", self.record("judgement.questions.v1", task_completed={"noul": 0.2}))
got = store.load(d, "job", "t")
self.assertEqual(got["questions"], "judgement.questions.v1"); self.assertFalse(store.is_current(got)) # only v1: shown, marked earlier
self.assertEqual(store.for_job(d, "job")["t"]["questions"], "judgement.questions.v1")
self.assertEqual(store.for_job(d, "job", {"t": None})["t"]["questions"], "judgement.questions.v1")
store.save(d, "job", "t", self.record(questions.VERSION, task_completed={"noul": 0.9}))
data = json.loads((self.tmp / "judgements" / "job" / "t.json").read_text())
self.assertEqual(sorted(data), ["judgement.questions.v1", "judgement.questions.v2"]) # both kept in one file
self.assertEqual(data["judgement.questions.v1"]["answers"]["task_completed"]["noul"], 0.2)
got = store.load(d, "job", "t")
self.assertEqual((got["questions"], got["answers"]["task_completed"]["noul"]), ("judgement.questions.v2", 0.9)); self.assertTrue(store.is_current(got))
self.assertEqual(store.load(d, "job", "t", version="judgement.questions.v1")["answers"]["task_completed"]["noul"], 0.2)
self.assertEqual(store.for_job(d, "job")["t"]["questions"], "judgement.questions.v2")
self.assertEqual(store.for_job(d, "job", {"t": None})["t"]["questions"], "judgement.questions.v2")
self.assertFalse(store.is_current(None))

def test_a_record_is_the_try_it_judged_and_no_retry_under_its_name(self):
Expand All @@ -252,7 +253,7 @@ def test_a_record_is_the_try_it_judged_and_no_retry_under_its_name(self):
store.save(d, "job", "t", self.record(questions.VERSION, task_completed={"noul": 0.2}), started=first)
self.assertEqual(store.load(d, "job", "t", started=first)["trial_started_at"], first)
self.assertIsNone(store.load(d, "job", "t", started=retry))
self.assertEqual(store.for_job(d, "job", started={"t": retry}), {})
self.assertEqual(store.for_job(d, "job", {"t": retry}), {})
self.assertIsNotNone(store.load(d, "job", "t")) # a read that names no start takes it as it is
store.save(d, "job", "t", self.record(questions.VERSION, failure_cause={"choice": "x"}), started=retry)
got = store.load(d, "job", "t", started=retry)
Expand Down Expand Up @@ -310,7 +311,7 @@ def chat(url, body, key):
self.assertEqual(store.load(str(self.tmp), "job", "t", lane="llm")["answers"], {"task_completed": {"noul": 0.3}})
self.assertIsNone(store.load(str(self.tmp), "job", "t")) # Decision's is its own
self.assertTrue((self.tmp / "judgements-llm" / "job" / "t.json").is_file())
self.assertEqual(store.for_job(str(self.tmp), "job"), {}); self.assertEqual(list(store.for_job(str(self.tmp), "job", lane="llm")), ["t"])
self.assertEqual(store.for_job(str(self.tmp), "job", {"t": None}), {}); self.assertEqual(list(store.for_job(str(self.tmp), "job", {"t": None}, lane="llm")), ["t"])


ADVERSARIAL = HERE / "fixtures" / "judge" / "adversarial-trajectory.json"
Expand Down
4 changes: 2 additions & 2 deletions app/tests/test_judge_cost.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ def test_the_store_walks_every_record_and_every_version_and_calibrates(self):
(self.tmp / "judgements" / "job-a" / "broken.json").write_text("{not json")
(self.tmp / "judgements" / "job-a" / "notes.txt").write_text("x")
seen = sorted((j, t, v) for j, t, v, _ in store.all_records(self.d))
self.assertEqual(seen, [("job-a", "t1", "judgement.questions.v1"), ("job-a", "t1", "judgement.questions.v2"), ("job-b", "t_2", "judgement.questions.v2")])
self.assertEqual([t for _, t, _, _ in store.all_records(self.d, "job-b")], ["t_2"])
self.assertEqual(seen, [("job-a", "t1", "judgement.questions.v1"), ("job-a", "t1", "judgement.questions.v2"), ("job-b", store.key("t/2"), "judgement.questions.v2")])
self.assertEqual([t for _, t, _, _ in store.all_records(self.d, "job-b")], [store.key("t/2")])
self.assertEqual(store.calibration(self.d), {"chars_per_token": 3.25, "n": 2, "low": 2.5, "high": 3.5, "n_large": 0}) # small samples only: the band widened to the default

def test_truncation_is_counted_per_state_kind_so_the_cut_state_can_be_named(self):
Expand Down
60 changes: 60 additions & 0 deletions app/tests/test_judge_store_keys.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
"""Two different trials, or two different jobs, never share one judgement
file; a store written under the older names is still read; and a job's
judged trials are its own trials only."""
from __future__ import annotations

import json
import os
import shutil
import tempfile
import unittest

from nullius.judge import questions as Q, store


def rec(choice):
return {"questions": Q.VERSION, "answers": {"q": {"choice": choice}}, "judge": {"model": "m"}}


class StoreKeys(unittest.TestCase):
def setUp(self):
self.ws = tempfile.mkdtemp(prefix="nullius-judge-keys-")
self.addCleanup(shutil.rmtree, self.ws, True)

def test_two_task_names_keep_two_records(self):
store.save(self.ws, "task:ds", "task:abc", rec("first"))
store.save(self.ws, "task:ds", "task_abc", rec("second"))
self.assertEqual(store.load(self.ws, "task:ds", "task:abc")["answers"]["q"]["choice"], "first")
self.assertEqual(store.load(self.ws, "task:ds", "task_abc")["answers"]["q"]["choice"], "second")

def test_a_split_agent_view_and_a_job_of_the_lookalike_name_keep_two_records(self):
store.save(self.ws, "job~a-b", "t1", rec("view")) # parse.py names an agent's view <job>~<label>
store.save(self.ws, "job_a-b", "t1", rec("job"))
self.assertEqual(store.load(self.ws, "job~a-b", "t1")["answers"]["q"]["choice"], "view")

def test_a_safe_name_keeps_its_file_name(self):
store.save(self.ws, "job-a", "task__x1", rec("x"))
self.assertTrue(os.path.isfile(os.path.join(self.ws, "judgements", "job-a", "task__x1.json")))

def test_a_record_kept_under_the_older_name_is_read_and_merged_into(self):
older = os.path.join(self.ws, "judgements", "job_a-b", "t_1.json") # job~a-b, t:1 as the store spelled them before
os.makedirs(os.path.dirname(older))
with open(older, "w", encoding="utf-8") as fh:
json.dump({Q.VERSION: rec("kept"), Q.EARLIER[0]: {**rec("v1"), "questions": Q.EARLIER[0]}}, fh)
self.assertEqual(store.load(self.ws, "job~a-b", "t:1")["answers"]["q"]["choice"], "kept")
self.assertEqual(list(store.for_job(self.ws, "job~a-b", {"t:1": None})), ["t:1"])
self.assertEqual(store.job_dirs(self.ws, ["job~a-b"]), [os.path.dirname(older)])
store.save(self.ws, "job~a-b", "t:1", {**rec("later"), "answers": {"r": {"choice": "y"}}})
got = store.load(self.ws, "job~a-b", "t:1")
self.assertEqual(sorted(got["answers"]), ["q", "r"])
self.assertEqual(store.load(self.ws, "job~a-b", "t:1", Q.EARLIER[0])["answers"]["q"]["choice"], "v1")

def test_a_v1_pair_record_is_not_a_judged_trial_of_job_a(self):
store.save(self.ws, "job-a", "t1", rec("trial"))
store.save(self.ws, "job-a", store.pair_key("t1", "t9"), {**rec("A"), "questions": Q.EARLIER[0]})
self.assertEqual(sorted(store.for_job(self.ws, "job-a", {"t1": None, "t2": None})), ["t1"])
self.assertEqual(store.load(self.ws, "job-a", store.pair_key("t1", "t9"))["answers"]["q"]["choice"], "A")


if __name__ == "__main__":
unittest.main()
6 changes: 3 additions & 3 deletions app/tests/test_reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,10 @@ def test_other_is_listed_apart_and_a_v1_record_still_reconciles(self):
self.assertEqual([r["task"] for r in out["rows"]["other"]], ["failed"]); self.assertEqual(out["rows"]["disagree"][0]["task"], "setup")
self.assertEqual(out["rows"]["disagree"][0]["version"], "judgement.questions.v1"); self.assertEqual(out["counts"]["other"], 1)

def test_the_stored_name_is_mapped_through_key_and_nothing_is_written(self):
recs = {"KEY:failed__x": record("environment_error", 0.9)}
def test_a_record_is_found_by_its_trial_name_and_nothing_is_written(self):
recs = {"failed__x": record("environment_error", 0.9)}
before = [(t.trial_name, t.outcome, t.reward) for t in self.job.trials]
out = R.reconcile(self.job, recs, key=lambda n: "KEY:" + n)
out = R.reconcile(self.job, recs)
self.assertEqual(out["rows"]["disagree"][0]["task"], "failed")
self.assertEqual([(t.trial_name, t.outcome, t.reward) for t in self.job.trials], before) # the trials are read, never written
self.assertEqual(self.by_task["odd"].outcome, "unknown")
Expand Down
Loading
Loading