Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions app/nullius/importers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
from __future__ import annotations

import importlib
import os
import pkgutil
import shutil
import tempfile
Expand Down Expand Up @@ -136,5 +137,27 @@ def problem(path: str) -> str | None:
return None


def write_if_changed(path: str, data: bytes) -> bool:
"""Write *data* to *path* unless the file already holds it; True when
written. A conversion runs at each open of a workspace; an unchanged one
keeps its mtimes, and with them the parse cache that keys on them."""
try:
with open(path, "rb") as fh:
if fh.read() == data:
return False
except OSError:
pass
with open(path, "wb") as fh:
fh.write(data)
return True


def mark_changed(job_dir: str, summaries: list) -> None:
"""Move the job's own ``result.json`` when one of its trials was written:
an ended job's cached parse is checked by that file, not trial by trial."""
if any(s["changed"] for s in summaries):
os.utime(os.path.join(job_dir, "result.json"))


def labels() -> str:
return ", ".join(getattr(m, "LABEL", name) for name, m in available().items())
50 changes: 27 additions & 23 deletions app/nullius/importers/atif.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import shutil
from datetime import timezone

from nullius import importers
from nullius.redact import redact
from nullius.traces.trajectory import parse_ts

Expand Down Expand Up @@ -273,6 +274,10 @@ def number(key):
}


def _dump(path: str, data) -> bool:
return importers.write_if_changed(path, json.dumps(data, indent=2).encode())


def _sha256(path: str) -> str:
digest = hashlib.sha256()
with open(path, "rb") as fh:
Expand Down Expand Up @@ -316,15 +321,15 @@ def convert_document(path: str, doc: dict, trial: str, job_dir: str) -> dict:
"session_id": doc.get("session_id"),
},
}
with open(os.path.join(trial_dir, "result.json"), "w", encoding="utf-8") as fh:
json.dump(result, fh, indent=2)
changed = _dump(os.path.join(trial_dir, "result.json"), result)

# Byte for byte: the trial page reads what the producer wrote, and a
# digest of the copy is a digest of the source.
shutil.copyfile(path, os.path.join(agent_dir, "trajectory.json"))
with open(path, "rb") as fh:
changed |= importers.write_if_changed(os.path.join(agent_dir, "trajectory.json"), fh.read())

return {"trial_dir": trial, "started_at": started, "finished_at": finished,
"agent_info": result["agent_info"]}
"agent_info": result["agent_info"], "changed": changed}



Expand All @@ -346,30 +351,29 @@ def _write_job(docs: list, root: str, out_root: str, job_name: str, command: str
# ordered as instants, not as text: one stamp with a zone and one without sort apart as strings
starts = sorted((s["started_at"] for s in summaries if s["started_at"]), key=_instant)
ends = sorted((s["finished_at"] for s in summaries if s["finished_at"]), key=_instant)
with open(os.path.join(job_dir, "result.json"), "w", encoding="utf-8") as fh:
json.dump({
"started_at": starts[0] if starts else None,
"finished_at": ends[-1] if ends else None,
"n_total_trials": len(summaries),
"stats": {"n_errored_trials": 0, "n_running_trials": 0,
"n_pending_trials": 0, "n_retries": 0},
}, fh, indent=2)
_dump(os.path.join(job_dir, "result.json"), {
"started_at": starts[0] if starts else None,
"finished_at": ends[-1] if ends else None,
"n_total_trials": len(summaries),
"stats": {"n_errored_trials": 0, "n_running_trials": 0,
"n_pending_trials": 0, "n_retries": 0},
})
importers.mark_changed(job_dir, summaries)

# The first document's producer names the job; a tree that mixes
# producers still shows each trial's own under agent_info.
agent = next((s["agent_info"] for s in summaries if s["agent_info"]["name"]),
summaries[0]["agent_info"])
with open(os.path.join(job_dir, "config.json"), "w", encoding="utf-8") as fh:
json.dump({
"job_name": job_name,
"harness": NAME, # provenance; the walker ignores unknown keys
"command": command or (
f"PYTHONPATH=app python3 -m nullius.importers.cli {shlex.quote(root)} "
f"-o {shlex.quote(out_root)} --job-name {shlex.quote(job_name)}"),
"agents": [{"name": agent["name"],
"model_name": agent["model_info"]["name"]}],
"tasks": [{"name": f"{NAME}/{s['trial_dir']}"} for s in summaries],
}, fh, indent=2)
_dump(os.path.join(job_dir, "config.json"), {
"job_name": job_name,
"harness": NAME, # provenance; the walker ignores unknown keys
"command": command or (
f"PYTHONPATH=app python3 -m nullius.importers.cli {shlex.quote(root)} "
f"-o {shlex.quote(out_root)} --job-name {shlex.quote(job_name)}"),
"agents": [{"name": agent["name"],
"model_name": agent["model_info"]["name"]}],
"tasks": [{"name": f"{NAME}/{s['trial_dir']}"} for s in summaries],
})
return job_dir


Expand Down
14 changes: 7 additions & 7 deletions app/nullius/importers/otel.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import shutil
from datetime import datetime, timezone

from nullius import importers
from nullius.redact import redact
from nullius.traces import otel as reader
from nullius.traces import trace
Expand Down Expand Up @@ -139,10 +140,8 @@ def _iso(ns: int) -> str:
return datetime.fromtimestamp(ns / 1e9, tz=timezone.utc).isoformat().replace("+00:00", "Z")


def _dump(path: str, data) -> None:
with open(path, "w", encoding="utf-8") as fh:
json.dump(data, fh, ensure_ascii=False, indent=1)
fh.write("\n")
def _dump(path: str, data) -> bool:
return importers.write_if_changed(path, (json.dumps(data, ensure_ascii=False, indent=1) + "\n").encode())


def _window(document: dict) -> tuple:
Expand Down Expand Up @@ -206,10 +205,10 @@ def write_trial(document: dict, job_dir: str) -> dict:
"n_cache_tokens": tokens.get("cache_read_tokens", 0),
"cost_usd": None,
}
_dump(os.path.join(trial_dir, "result.json"), result)
_dump(os.path.join(agent_dir, "trace.json"), document)
changed = _dump(os.path.join(trial_dir, "result.json"), result)
changed |= _dump(os.path.join(agent_dir, "trace.json"), document)
return {"trial_dir": trial, "started_at": started, "finished_at": finished,
"model": model, "agent": agent.get("name") or ""}
"model": model, "agent": agent.get("name") or "", "changed": changed}


def _write_job(documents: list, out_root: str, job_name: str, command: str,
Expand All @@ -235,6 +234,7 @@ def _write_job(documents: list, out_root: str, job_name: str, command: str,
"stats": {"n_errored_trials": 0, "n_running_trials": 0,
"n_pending_trials": 0, "n_retries": 0},
})
importers.mark_changed(job_dir, summaries)
agents = sorted({s["agent"] for s in summaries if s["agent"]}) or [""]
models = sorted({s["model"] for s in summaries if s["model"]})
_dump(os.path.join(job_dir, "config.json"), {
Expand Down
66 changes: 66 additions & 0 deletions app/tests/test_import_unchanged_source.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
"""Opening a workspace again over an unchanged ATIF or OpenTelemetry source
leaves the converted job as it was, so its cached parse is used; a changed
source is converted and parsed again."""
from __future__ import annotations

import os
import shutil
import tempfile
import time
import unittest
from pathlib import Path

from nullius import importers, workspace
from tests.harnesses.test_atif import minimal, write_doc
from tests.test_cli_contract import otlp_doc


def open_again(root, imports):
"""What shared._import_roots does for a root on each open_workspace."""
module = importers.detect(root)
return [d for tree in importers.trees(module, root) for d in module.convert(tree, imports)]


def mtimes(job_dir):
return {os.path.relpath(os.path.join(d, f), job_dir): os.stat(os.path.join(d, f)).st_mtime_ns
for d, _, files in os.walk(job_dir) for f in files}


class UnchangedSource(unittest.TestCase):
def setUp(self):
self.tmp = Path(tempfile.mkdtemp(prefix="nullius-import-"))
self.addCleanup(shutil.rmtree, self.tmp, True)
self.src = self.tmp / "src"

def check(self, change):
"""Convert, read, open again unchanged, then again after *change*."""
ws = workspace.create(str(self.tmp / "workspaces"), [str(self.src)], "w")
[job_dir] = open_again(str(self.src), ws.imports_dir)
before = mtimes(job_dir)
self.assertFalse(workspace.load_job(ws, job_dir)[1]) # the first read parses
time.sleep(0.05)
open_again(str(self.src), ws.imports_dir) # opened again; the source unchanged
moved = sorted(f for f, m in mtimes(job_dir).items() if before.get(f) != m)
self.assertEqual((moved, workspace.load_job(ws, job_dir)[1]), ([], True))
change()
open_again(str(self.src), ws.imports_dir) # opened again; the source changed
self.assertNotEqual(mtimes(job_dir), before)
self.assertFalse(workspace.load_job(ws, job_dir)[1])

# Each change leaves the job's own result.json and config.json as they were.
def test_atif(self):
write_doc(self.src, "run-1.json", minimal())
edited = minimal()
edited["steps"][1]["message"] = "redone"
self.check(lambda: write_doc(self.src, "run-1.json", edited))

def test_opentelemetry(self):
write_doc(self.src, "export.json", otlp_doc())
edited = otlp_doc()
edited["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"] = [
{"key": "k", "value": {"stringValue": "v"}}]
self.check(lambda: write_doc(self.src, "export.json", edited))


if __name__ == "__main__":
unittest.main()
Loading