From 30230fe0b91b1bcd8e5a5b78e8693bf138339b39 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Fri, 20 Nov 2026 12:12:38 +0100 Subject: [PATCH] fix(judge): give every job and trial a judgement file of its own The store spelled a file name by replacing every unsafe character with `_`, so `task:abc` and `task_abc`, or the agent view `job~a-b` and a job named `job_a-b`, wrote one file and one judgement replaced the other. `for_job` listed every file under a job's folder, so a v1 pair record counted as a judged trial on the job page. `store.key` keeps a name that is already safe as it is, and adds a short digest of the name to its safe spelling otherwise; no safe spelling holds a `~`, so two names never meet. Reads and the merge of a save fall back to the older spelling, so existing stores are read as they are: no file is renamed or rewritten until it is judged again, and that save writes the new name. `for_job` takes the job's trials and returns their records by trial name, so the pages and the reconcile view no longer spell keys themselves and a pair record is never a trial. --- app/nullius/judge/lanes.py | 5 +- app/nullius/judge/reconcile.py | 9 ++-- app/nullius/judge/store.py | 82 +++++++++++++++++------------- app/tests/test_judge.py | 13 ++--- app/tests/test_judge_cost.py | 4 +- app/tests/test_judge_store_keys.py | 60 ++++++++++++++++++++++ app/tests/test_reconcile.py | 6 +-- app/web/judge.py | 20 ++++---- app/web/reconcile.py | 4 +- 9 files changed, 136 insertions(+), 67 deletions(-) create mode 100644 app/tests/test_judge_store_keys.py diff --git a/app/nullius/judge/lanes.py b/app/nullius/judge/lanes.py index 2365f97a..bd7a5a2e 100644 --- a/app/nullius/judge/lanes.py +++ b/app/nullius/judge/lanes.py @@ -38,7 +38,7 @@ _LINE_ORDER = {AGREE_LANE: 0, DISAGREE_LANE: 1, GUTTER: 2} -def board(job, records: dict, threshold: float = R.THRESHOLD, key=None) -> dict | None: +def board(job, records: dict, threshold: float = R.THRESHOLD) -> dict | None: """The lanes for one job, or None for a control (nothing to reconcile). Returns ``rows`` (one per outcome present, in the classifier's order, @@ -48,8 +48,7 @@ def board(job, records: dict, threshold: float = R.THRESHOLD, key=None) -> dict trial) and which lanes any line reaches.""" if not R.applies(job): return None - key = key or (lambda name: name) - rows = {t.trial_name: R.row_of(t, records.get(key(t.trial_name)), threshold) for t in job.trials} + rows = {t.trial_name: R.row_of(t, records.get(t.trial_name), threshold) for t in job.trials} status = {} out = [] for label in classify.ORDER: diff --git a/app/nullius/judge/reconcile.py b/app/nullius/judge/reconcile.py index 307ee2c9..f02da290 100644 --- a/app/nullius/judge/reconcile.py +++ b/app/nullius/judge/reconcile.py @@ -154,14 +154,11 @@ def _rank(row: dict) -> tuple: return (-(row["confidence"] or 0.0), -(row["p"] or 0.0), row["trial_name"]) -def reconcile(job, records: dict, threshold: float = THRESHOLD, key=None) -> dict: +def reconcile(job, records: dict, threshold: float = THRESHOLD) -> dict: """The job's reconciliation: rows by status (each list ranked strongest first), and the counts the page states. `records` is the store's - {stored name: record}; `key` maps a trial name to its stored name (the - store's own sanitiser), identity when not given. Reads the job; writes - nothing.""" - key = key or (lambda name: name) - rows = [row_of(t, records.get(key(t.trial_name)), threshold) for t in job.trials] + {trial name: record}. Reads the job; writes nothing.""" + rows = [row_of(t, records.get(t.trial_name), threshold) for t in job.trials] by_status = {s: sorted((r for r in rows if r["status"] == s), key=_rank) for s in STATUSES} judged = sum(1 for r in rows if r["choice"] is not None) asked = sum(1 for t in job.trials if t.outcome not in (classify.RESOLVED, classify.SCORED)) diff --git a/app/nullius/judge/store.py b/app/nullius/judge/store.py index b6f1b232..a7d47365 100644 --- a/app/nullius/judge/store.py +++ b/app/nullius/judge/store.py @@ -1,5 +1,6 @@ """Where a judgement lives: the workspace's own store, beside the parsed -jobs - ``/judgements//.json`` for System +jobs - ``/judgements//.json`` (each name +spelled by `key`) for System One's, ``/judgements-llm/...`` for the LLM judge's (a trial has both, apart, ``lane``) - keyed by question version inside. Never under the job's ``agent/`` directory: the run tree is Harbor's, and a re-parse of the job does not touch this. @@ -11,6 +12,7 @@ carries - and a write always lands under the version the record names.""" from __future__ import annotations +import hashlib import json import os import re @@ -39,8 +41,36 @@ def _root(ws_dir: str, lane: str) -> str: return os.path.join(ws_dir, ROOTS.get(lane, "judgements")) -def _path(ws_dir: str, job_id: str, trial_name: str, lane: str = "decision") -> str: - return os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id), _SAFE.sub("_", trial_name) + ".json") +def key(name: str) -> str: + """A job id's or trial name's file name: the name itself when every + character is safe, else its safe spelling and a digest of the name, so + two names never share a file (no safe spelling holds a ``~``).""" + safe = _SAFE.sub("_", name) + return safe if safe == name else f"{safe}~{hashlib.sha256(name.encode('utf-8')).hexdigest()[:10]}" + + +def _names(name: str) -> tuple: + """`key`, then the safe spelling alone that a store written before `key` used, still read.""" + return tuple(dict.fromkeys((key(name), _SAFE.sub("_", name)))) + + +def _paths(ws_dir: str, job_id: str, trial_name: str, lane: str = "decision") -> list: + """Where a judgement is written, then where an older store may hold it.""" + root = _root(ws_dir, lane) + return [os.path.join(root, j, t + ".json") for j in _names(job_id) for t in _names(trial_name)] + + +def _read(paths) -> dict: + """The first judgement file of `paths` there is, read; {} when there is none or it cannot be read.""" + for path in paths: + try: + with open(path, encoding="utf-8") as fh: + return json.load(fh) or {} + except FileNotFoundError: + continue + except (OSError, ValueError): + return {} + return {} def _versions(version: Optional[str]) -> tuple: @@ -68,11 +98,7 @@ def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = Non """The record under `version`; with none named, the current version or the newest earlier one that has a record. `started`: the trial's own start, so a record of an earlier try under its name is none of its.""" - try: - with open(_path(ws_dir, job_id, trial_name, lane), encoding="utf-8") as fh: - return _pick(json.load(fh) or {}, version, started) - except (OSError, ValueError): - return None + return _pick(_read(_paths(ws_dir, job_id, trial_name, lane)), version, started) def is_current(record: Optional[dict]) -> bool: @@ -86,14 +112,11 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "d record's answers and hygiene readings merge over the version's earlier ones, so judging one question later does not drop the others. Saves to one file take turns; each writes through a temporary file of its own.""" - path = _path(ws_dir, job_id, trial_name, lane) + paths = _paths(ws_dir, job_id, trial_name, lane) + path = paths[0] os.makedirs(os.path.dirname(path), exist_ok=True) with _lock(path): - try: - with open(path, encoding="utf-8") as fh: - data = json.load(fh) or {} - except (OSError, ValueError): - data = {} + data = _read(paths) version = record.get("questions", Q.VERSION) earlier = data.get(version) or {} if not _of(earlier, started): @@ -114,33 +137,22 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "d return merged -def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "decision", - started: Optional[dict] = None) -> dict: - """Every judged trial of a job: {trial_name: record}, each the current - version's record or the newest earlier one, as `load` picks it. - `started`: {trial_name: its start}, as `load` takes it.""" - d = os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id)) +def for_job(ws_dir: str, job_id: str, started: dict, version: Optional[str] = None, lane: str = "decision") -> dict: + """The judged trials of a job: {trial_name: record} for each trial of + `started` - {trial_name: its start}, every trial of the job - that has + one, as `load` picks it. Nothing else under the job's name is a trial of + it: a v1 pair record is not.""" out = {} - try: - names = os.listdir(d) - except OSError: - return out - for n in names: - if not n.endswith(".json"): - continue - try: - with open(os.path.join(d, n), encoding="utf-8") as fh: - rec = _pick(json.load(fh) or {}, version, (started or {}).get(n[:-5])) - except (OSError, ValueError): - continue + for name, start in started.items(): + rec = load(ws_dir, job_id, name, version, lane, start) if rec: - out[n[:-5]] = rec + out[name] = rec return out def job_dirs(ws_dir: str, job_ids) -> list: """The folders that hold these jobs' judgements, in every lane.""" - return [d for lane in ROOTS for j in job_ids if os.path.isdir(d := os.path.join(_root(ws_dir, lane), _SAFE.sub("_", j)))] + return [d for lane in ROOTS for j in job_ids for n in _names(j) if os.path.isdir(d := os.path.join(_root(ws_dir, lane), n))] def count(ws_dir: str) -> int: @@ -154,7 +166,7 @@ def all_records(ws_dir: str, job_id: Optional[str] = None, lane: str = "decision version, record). One job's when `job_id` is given.""" root = _root(ws_dir, lane) try: - jobs = [_SAFE.sub("_", job_id)] if job_id else sorted(os.listdir(root)) + jobs = list(_names(job_id)) if job_id else sorted(os.listdir(root)) except OSError: return for j in jobs: diff --git a/app/tests/test_judge.py b/app/tests/test_judge.py index 74ffb859..6ba831f7 100644 --- a/app/tests/test_judge.py +++ b/app/tests/test_judge.py @@ -193,7 +193,7 @@ def test_save_load_merge_and_survive_a_reparse(self): rec = {"questions": "judgement.questions.v1", "judge": {"provider": "typesafe.ai", "model": "jev-latest"}, "asked_at": "2026-09-17T09:00:00Z", "state_chars": 10, "truncated": False, "answers": {"task_completed": {"noul": 0.9}}, "usage": {"input_tokens": 1, "output_tokens": 1}} store.save(self.ws.dir, "job-1", "task__abc/def", rec) - path = Path(self.ws.dir) / "judgements" / "job-1" / "task__abc_def.json" + path = Path(self.ws.dir) / "judgements" / "job-1" / (store.key("task__abc/def") + ".json") self.assertTrue(path.exists()); self.assertFalse((self.tmp / "jobs").exists()) # under the workspace, never the job tree self.assertEqual(store.load(self.ws.dir, "job-1", "task__abc/def")["answers"], {"task_completed": {"noul": 0.9}}) later = dict(rec, asked_at="2026-09-17T10:00:00Z", answers={"transcript_quality": {"score": 2.0, "legend": ["a", "b", "c"], "probabilities": {}, "confidence": 0.9}}) @@ -201,7 +201,8 @@ def test_save_load_merge_and_survive_a_reparse(self): got = store.load(self.ws.dir, "job-1", "task__abc/def") self.assertEqual(sorted(got["answers"]), ["task_completed", "transcript_quality"]); self.assertEqual(got["asked_at"], "2026-09-17T10:00:00Z") # answers merge per version self.assertIsNone(store.load(self.ws.dir, "job-1", "task__abc/def", version="judgement.questions.v3")) - self.assertEqual(list(store.for_job(self.ws.dir, "job-1")), ["task__abc_def"]); self.assertEqual(store.for_job(self.ws.dir, "job-2"), {}) + trials = {"task__abc/def": None, "other": None} + self.assertEqual(list(store.for_job(self.ws.dir, "job-1", trials)), ["task__abc/def"]); self.assertEqual(store.for_job(self.ws.dir, "job-2", trials), {}) (Path(self.ws.jobs_dir)).mkdir(parents=True, exist_ok=True); (Path(self.ws.jobs_dir) / "x.json").write_text("{}") workspace.clear_cache(self.ws) # a re-parse of the job self.assertEqual(store.load(self.ws.dir, "job-1", "task__abc/def")["answers"]["task_completed"], {"noul": 0.9}) @@ -233,7 +234,7 @@ def test_v1_and_v2_coexist_and_the_read_prefers_the_current(self): store.save(d, "job", "t", self.record("judgement.questions.v1", task_completed={"noul": 0.2})) got = store.load(d, "job", "t") self.assertEqual(got["questions"], "judgement.questions.v1"); self.assertFalse(store.is_current(got)) # only v1: shown, marked earlier - self.assertEqual(store.for_job(d, "job")["t"]["questions"], "judgement.questions.v1") + self.assertEqual(store.for_job(d, "job", {"t": None})["t"]["questions"], "judgement.questions.v1") store.save(d, "job", "t", self.record(questions.VERSION, task_completed={"noul": 0.9})) data = json.loads((self.tmp / "judgements" / "job" / "t.json").read_text()) self.assertEqual(sorted(data), ["judgement.questions.v1", "judgement.questions.v2"]) # both kept in one file @@ -241,7 +242,7 @@ def test_v1_and_v2_coexist_and_the_read_prefers_the_current(self): got = store.load(d, "job", "t") self.assertEqual((got["questions"], got["answers"]["task_completed"]["noul"]), ("judgement.questions.v2", 0.9)); self.assertTrue(store.is_current(got)) self.assertEqual(store.load(d, "job", "t", version="judgement.questions.v1")["answers"]["task_completed"]["noul"], 0.2) - self.assertEqual(store.for_job(d, "job")["t"]["questions"], "judgement.questions.v2") + self.assertEqual(store.for_job(d, "job", {"t": None})["t"]["questions"], "judgement.questions.v2") self.assertFalse(store.is_current(None)) def test_a_record_is_the_try_it_judged_and_no_retry_under_its_name(self): @@ -252,7 +253,7 @@ def test_a_record_is_the_try_it_judged_and_no_retry_under_its_name(self): store.save(d, "job", "t", self.record(questions.VERSION, task_completed={"noul": 0.2}), started=first) self.assertEqual(store.load(d, "job", "t", started=first)["trial_started_at"], first) self.assertIsNone(store.load(d, "job", "t", started=retry)) - self.assertEqual(store.for_job(d, "job", started={"t": retry}), {}) + self.assertEqual(store.for_job(d, "job", {"t": retry}), {}) self.assertIsNotNone(store.load(d, "job", "t")) # a read that names no start takes it as it is store.save(d, "job", "t", self.record(questions.VERSION, failure_cause={"choice": "x"}), started=retry) got = store.load(d, "job", "t", started=retry) @@ -310,7 +311,7 @@ def chat(url, body, key): self.assertEqual(store.load(str(self.tmp), "job", "t", lane="llm")["answers"], {"task_completed": {"noul": 0.3}}) self.assertIsNone(store.load(str(self.tmp), "job", "t")) # Decision's is its own self.assertTrue((self.tmp / "judgements-llm" / "job" / "t.json").is_file()) - self.assertEqual(store.for_job(str(self.tmp), "job"), {}); self.assertEqual(list(store.for_job(str(self.tmp), "job", lane="llm")), ["t"]) + self.assertEqual(store.for_job(str(self.tmp), "job", {"t": None}), {}); self.assertEqual(list(store.for_job(str(self.tmp), "job", {"t": None}, lane="llm")), ["t"]) ADVERSARIAL = HERE / "fixtures" / "judge" / "adversarial-trajectory.json" diff --git a/app/tests/test_judge_cost.py b/app/tests/test_judge_cost.py index 46763ff0..f38acb1b 100644 --- a/app/tests/test_judge_cost.py +++ b/app/tests/test_judge_cost.py @@ -92,8 +92,8 @@ def test_the_store_walks_every_record_and_every_version_and_calibrates(self): (self.tmp / "judgements" / "job-a" / "broken.json").write_text("{not json") (self.tmp / "judgements" / "job-a" / "notes.txt").write_text("x") seen = sorted((j, t, v) for j, t, v, _ in store.all_records(self.d)) - self.assertEqual(seen, [("job-a", "t1", "judgement.questions.v1"), ("job-a", "t1", "judgement.questions.v2"), ("job-b", "t_2", "judgement.questions.v2")]) - self.assertEqual([t for _, t, _, _ in store.all_records(self.d, "job-b")], ["t_2"]) + self.assertEqual(seen, [("job-a", "t1", "judgement.questions.v1"), ("job-a", "t1", "judgement.questions.v2"), ("job-b", store.key("t/2"), "judgement.questions.v2")]) + self.assertEqual([t for _, t, _, _ in store.all_records(self.d, "job-b")], [store.key("t/2")]) self.assertEqual(store.calibration(self.d), {"chars_per_token": 3.25, "n": 2, "low": 2.5, "high": 3.5, "n_large": 0}) # small samples only: the band widened to the default def test_truncation_is_counted_per_state_kind_so_the_cut_state_can_be_named(self): diff --git a/app/tests/test_judge_store_keys.py b/app/tests/test_judge_store_keys.py new file mode 100644 index 00000000..fb19b79e --- /dev/null +++ b/app/tests/test_judge_store_keys.py @@ -0,0 +1,60 @@ +"""Two different trials, or two different jobs, never share one judgement +file; a store written under the older names is still read; and a job's +judged trials are its own trials only.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest + +from nullius.judge import questions as Q, store + + +def rec(choice): + return {"questions": Q.VERSION, "answers": {"q": {"choice": choice}}, "judge": {"model": "m"}} + + +class StoreKeys(unittest.TestCase): + def setUp(self): + self.ws = tempfile.mkdtemp(prefix="nullius-judge-keys-") + self.addCleanup(shutil.rmtree, self.ws, True) + + def test_two_task_names_keep_two_records(self): + store.save(self.ws, "task:ds", "task:abc", rec("first")) + store.save(self.ws, "task:ds", "task_abc", rec("second")) + self.assertEqual(store.load(self.ws, "task:ds", "task:abc")["answers"]["q"]["choice"], "first") + self.assertEqual(store.load(self.ws, "task:ds", "task_abc")["answers"]["q"]["choice"], "second") + + def test_a_split_agent_view_and_a_job_of_the_lookalike_name_keep_two_records(self): + store.save(self.ws, "job~a-b", "t1", rec("view")) # parse.py names an agent's view ~