Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions app/nullius/judge/batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,9 @@ def ask_and_store(ws_dir: str, job_id: str, trial_name: str, states: dict, key,
calibration = store.calibration(ws_dir)
rec = ask_states(states, key, post=post, calibration=calibration)
if rec is not None:
# a call over some of the kinds keeps the earlier readings for the others, as the answers are kept
lane = getattr(key, "lane", "decision") # the LLM judge's records apart from Decision's
earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane, started=started) or {}
rec["hygiene"] = {**(earlier.get("hygiene") or {}), **rec.get("hygiene", {})}
store.save(ws_dir, job_id, trial_name, rec, lane=lane, started=started)
# the LLM judge's records apart from Decision's; a call over some of the kinds keeps the
# earlier readings for the others, as the answers are kept (`store.save`), and returns them
lane = getattr(key, "lane", "decision")
rec.setdefault("hygiene", {})
rec["hygiene"] = store.save(ws_dir, job_id, trial_name, rec, lane=lane, started=started)["hygiene"]
return rec
60 changes: 40 additions & 20 deletions app/nullius/judge/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,24 @@
import json
import os
import re
import tempfile
import threading
from typing import Optional

from . import client, questions as Q

_SAFE = re.compile(r"[^A-Za-z0-9._@+-]")


_LOCKS: dict = {} # one per judgement file: a save reads, merges and writes it whole
_LOCKS_GUARD = threading.Lock()


def _lock(path: str) -> threading.Lock:
with _LOCKS_GUARD:
return _LOCKS.setdefault(path, threading.Lock())


ROOTS = {"decision": "judgements", "llm": "judgements-llm"} # one directory per judgement


Expand Down Expand Up @@ -70,28 +81,37 @@ def is_current(record: Optional[dict]) -> bool:


def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "decision",
started: Optional[str] = None) -> None:
"""The record under its question version; other versions kept. A
record's answers merge over the version's earlier answers, so judging
one question later does not drop the others."""
started: Optional[str] = None) -> dict:
"""The record under its question version, returned as stored; other versions kept. A
record's answers and hygiene readings merge over the version's earlier
ones, so judging one question later does not drop the others. Saves to
one file take turns; each writes through a temporary file of its own."""
path = _path(ws_dir, job_id, trial_name, lane)
os.makedirs(os.path.dirname(path), exist_ok=True)
try:
with open(path, encoding="utf-8") as fh:
data = json.load(fh) or {}
except (OSError, ValueError):
data = {}
version = record.get("questions", Q.VERSION)
earlier = data.get(version) or {}
if not _of(earlier, started):
earlier = {} # an earlier try's: none of its answers are this one's
merged = dict(record, **({"trial_started_at": started} if started else {}))
merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})}
data[version] = merged
tmp = path + ".tmp"
with open(tmp, "w", encoding="utf-8") as fh:
json.dump(data, fh, indent=1)
os.replace(tmp, path)
with _lock(path):
try:
with open(path, encoding="utf-8") as fh:
data = json.load(fh) or {}
except (OSError, ValueError):
data = {}
version = record.get("questions", Q.VERSION)
earlier = data.get(version) or {}
if not _of(earlier, started):
earlier = {} # an earlier try's: none of its answers are this one's
merged = dict(record, **({"trial_started_at": started} if started else {}))
merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})}
if "hygiene" in earlier or "hygiene" in record:
merged["hygiene"] = {**(earlier.get("hygiene") or {}), **(record.get("hygiene") or {})}
data[version] = merged
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=os.path.basename(path) + ".", suffix=".tmp")
try:
with os.fdopen(fd, "w", encoding="utf-8") as fh:
json.dump(data, fh, indent=1)
os.replace(tmp, path)
except BaseException:
os.unlink(tmp)
raise
return merged


def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "decision",
Expand Down
78 changes: 78 additions & 0 deletions app/tests/test_judge_store_race.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""Two saves to one trial's judgement that overlap both land in the file."""
from __future__ import annotations

import json
import shutil
import tempfile
import threading
import unittest
from unittest import mock

from nullius.judge import store


class OverlappingSaves(unittest.TestCase):
def test_two_overlapping_saves_keep_both_answers(self):
ws = tempfile.mkdtemp(prefix="nullius-judge-race-")
self.addCleanup(shutil.rmtree, ws, True)
both_read = threading.Barrier(2, timeout=2)
real_load = json.load

def load_then_wait(fh, *a, **k):
data = real_load(fh, *a, **k)
try:
both_read.wait() # each save has read the file before either writes, unless a lock keeps them apart
except threading.BrokenBarrierError:
pass
return data

store.save(ws, "job", "t1", {"questions": "v1", "answers": {}})
errors = []

def save(question):
try:
store.save(ws, "job", "t1", {"questions": "v1", "answers": {question: "yes"}})
except Exception as e:
errors.append(repr(e))

with mock.patch.object(store.json, "load", load_then_wait):
threads = [threading.Thread(target=save, args=(q,)) for q in ("q1", "q2")]
for t in threads:
t.start()
for t in threads:
t.join()
answers = store.load(ws, "job", "t1", "v1")["answers"]
self.assertEqual((errors, sorted(answers)), ([], ["q1", "q2"]))

def test_saves_to_two_trials_do_not_wait_on_each_other(self):
ws = tempfile.mkdtemp(prefix="nullius-judge-race-")
self.addCleanup(shutil.rmtree, ws, True)
for t in ("t1", "t2"):
store.save(ws, "job", t, {"questions": "v1", "answers": {}})
both_read = threading.Barrier(2, timeout=5)
real_load = json.load

def load_then_wait(fh, *a, **k):
data = real_load(fh, *a, **k)
both_read.wait() # raises if one save holds the other back
return data

errors = []

def save(trial):
try:
store.save(ws, "job", trial, {"questions": "v1", "answers": {"q": "yes"}})
except Exception as e:
errors.append(repr(e))

with mock.patch.object(store.json, "load", load_then_wait):
threads = [threading.Thread(target=save, args=(t,)) for t in ("t1", "t2")]
for t in threads:
t.start()
for t in threads:
t.join()
self.assertEqual(errors, [])


if __name__ == "__main__":
unittest.main()
Loading