diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index 73408ddc9..b3140f3de 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -30,6 +30,7 @@ "src/inferrsi/apps/workflows/benchmark_compare.py", "tests/check_installed.py", "tests/test_apps_assess.py", + "tests/test_apps_evidence_capture.py", "tests/test_apps_module_migrations.py", "tests/test_apps_output.py", "tests/test_apps_port_validation.py", @@ -61,6 +62,7 @@ "tests/check_installed.py", "tests/test_apps_artifact_reader.py", "tests/test_apps_assess.py", + "tests/test_apps_evidence_capture.py", "tests/test_apps_generation_plan.py", "tests/test_apps_module_migrations.py", "tests/test_benchmark_assessment.py", @@ -88,6 +90,7 @@ "src/inferrsi/apps/delivery/adapters.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", + "src/inferrsi/apps/handoff/evidence_capture.py", "src/inferrsi/apps/result.py", "src/inferrsi/apps/storage/archive.py", "src/inferrsi/apps/storage/artifact_reader.py", diff --git a/docs/architecture.md b/docs/architecture.md index d33f51d02..9420edb45 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -199,6 +199,7 @@ src/inferrsi/ bridge.py Bind preparations to specs and fixed records to evidence context_provider.py Deliver Knowledge bytes and pins for Harness sealing candidate_provider.py Plan generation and collect fixed candidate outputs + evidence_capture.py Read H-authorized bytes by role for an injected proof projector storage/ archive.py Immutable preparations/assessments and manifest.json/result.json artifact_reader.py Read verified artifacts through authorized mounts diff --git a/docs/contracts/apps-evidence-capture.md b/docs/contracts/apps-evidence-capture.md new file mode 100644 index 000000000..95bac57db --- /dev/null +++ b/docs/contracts/apps-evidence-capture.md @@ -0,0 +1,55 @@ +# Apps evidence capture assembly + +`inferrsi.apps.api.EvidenceCapture` is defined in +`apps/handoff/evidence_capture.py`. It is the thin host adapter accepted in +[#565](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7064458592). +It imports no Benchmark business module, performs no grading and writes no +archive or RunStore. It passes the injected projector's result through unchanged. + +## Host configuration + +After reading the fixed preparation and protocol through their existing archive +ports, the host constructs the supported B projector and configures role mapping: + +```python +from inferrsi.apps.api import EvidenceCapture +from inferrsi.benchmark.proofs import ProofProjector +from inferrsi.harness.api import Harness + +projector = ProofProjector(prepared, protocol_bytes) +capture = EvidenceCapture( + projector.project, + candidate_role=prepared["candidate_contract"]["input_role"], + report_role=prepared["recipe"]["outputs"][0]["role"], +) +harness = Harness(store, target, artifact_reader=reader, evidence_capture=capture) +``` + +The supported frozen P1 recipe has exactly one report output. Roles come from +validated frozen metadata; Apps defines no task name, report schema, predicate, +sampler or producer policy. The normal Apps execute port submits its saved +request to `harness.execute`. Existing callers can keep their configured ports. + +## Data and failure boundaries + +The callback snapshots H facts, selects this attempt's configured candidate +input, and reads only through the H capability. In finalizing it also selects an +available configured report belonging to this attempt and its enclosing step. +An absent family is passed as explicit `None`; multiple sources are refused as +IdentityMismatch instead of choosing the first or last. A configured candidate +role that differs from actual facts is also refused. + +Opened streams are closed, including failed reads. Reads are bounded to size+1 +and bytes are independently checked against fixed Ref size/hash. Capability +refusals keep their original portable code for H's lifecycle handling. Unknown +host/type errors propagate; they do not become fabricated report absence. + +The injected B projector receives only supplied bytes and a defensive context +snapshot. It owns proof construction and independent family outcomes. H validates +the returned drafts, retains raw/derived bytes and seals their actual provenance +before its record commit. Apps never adds a check to a committed record. + +Standard manifest/result publication, assessment and view/trace all retain the +same committed RecordRef. Output publication failure preserves that reference; +replay uses the original attempt and skips capture. These mechanics tests do not +accept a real task/grader/sampler specification or missing measurement evidence. diff --git a/docs/module-layout.md b/docs/module-layout.md index d84c22a44..01fbbc272 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -52,6 +52,11 @@ Knowledge bytes and pins for Harness sealing. `candidate_provider.py` plans generation and collects candidates from fixed Harness records. These modules consume `apps/storage/`; storage never imports back into handoff. +`evidence_capture.py` defines the new `apps.api.EvidenceCapture` host adapter. +It reads H-authorized candidate/report bytes by configured role and passes them +to an injected pure projector; H owns proof sealing and the one record commit. +There is no former flat-path alias for this new capability. + The three former Apps paths retain explicit public exports recorded in the [migration inventory](apps-module-migrations.json). `apps.api.CandidateProvider` and the former provider module export the one class defined in diff --git a/docs/operations.md b/docs/operations.md index 2c5809e77..d9d9a206b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -55,6 +55,7 @@ Effect classes used below: | Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) | | Verify and execute one frozen source evaluation entry | Python `inferrsi.harness.api.Harness.execute` with trusted `LocalCpuTarget(module_root=...)` | Experiment-executing / record-writing | Fixed preparation and expected code digest; actual source captured after registration; T37 refusals start no worker; [source-entry contract](contracts/harness-entry-verification.md) | | Capture and seal evaluation proofs | Python `Harness(..., evidence_capture=...)`; [capture contract](contracts/harness-evidence-capture.md) | Read-only capture callback / record-writing | H authorizes attempt bytes, seals actual provenance and retains drafts before one commit; ordinary mode and replay never call the observer | +| Assemble evidence capture by role | Python `apps.api.EvidenceCapture`; [host adapter](contracts/apps-evidence-capture.md) | Read-only byte assembly / injected pure projection | Reads only H-authorized candidate/report refs; no grading, producer sealing or record/archive writes; reentry is short-circuited by H | | Replay the same saved execution request | Python `inferrsi.harness.api.Harness.execute` | Existing-state query / fixed record-reading | Same attempt and canonical digest; existing state short-circuits fresh binding/execution; [replay and original limits](contracts/harness-evaluation-replay.md) | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Assess one fixed P1 record offline | Python `benchmark.assessment.assess`; [production assess](contracts/production-assess.md) | Read-only retained evidence / pure encoding | Verifies the frozen protocol, actual capture and proof bytes; returns independent claims without launching a worker or writing an archive; Apps selects the primary key and archives the result | diff --git a/src/inferrsi/apps/api.py b/src/inferrsi/apps/api.py index 0b137ee57..e466a2e3c 100644 --- a/src/inferrsi/apps/api.py +++ b/src/inferrsi/apps/api.py @@ -17,6 +17,7 @@ verify_protocol, ) from .handoff.candidate_provider import CandidateProvider as CandidateProvider +from .handoff.evidence_capture import EvidenceCapture as EvidenceCapture from .storage.archive import ( ResourceReader, RunOutput, diff --git a/src/inferrsi/apps/handoff/evidence_capture.py b/src/inferrsi/apps/handoff/evidence_capture.py new file mode 100644 index 000000000..fe1a6c9be --- /dev/null +++ b/src/inferrsi/apps/handoff/evidence_capture.py @@ -0,0 +1,81 @@ +"""Read attempt sources by configured role and invoke a host-supplied pure projector.""" + +from collections.abc import Callable +from contextlib import closing +from copy import deepcopy +from hashlib import sha256 + +from inferrsi.harness.errors import PublicError + +from ..storage.result import validate_artifact_ref + + +def _one(values): + if len(values) > 1: + raise PublicError("IdentityMismatch", stage="evidence_capture", + message="The configured capture role has multiple sources.") + return values[0] if values else None + + +def _read(ref, artifacts): + if ref is None: + return None + validate_artifact_ref(ref) + fixed = deepcopy(ref) + opened = artifacts.open(deepcopy(fixed)) + if isinstance(opened, bytes): + data = opened + else: + chunks, remaining = [], fixed["size_bytes"] + 1 + with closing(opened): + while remaining: + chunk = opened.read(min(remaining, 1024 * 1024)) + if not isinstance(chunk, bytes): + raise TypeError("A capture capability must supply binary bytes or streams.") + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + if len(data) != fixed["size_bytes"] or sha256(data).hexdigest() != fixed["sha256"]: + raise PublicError("ArtifactIntegrityError", stage="evidence_capture", + message="Capture bytes do not match their retained reference.") + return data + + +class EvidenceCapture: + """Apps host adapter; roles and pure projection are configured from frozen B metadata. + + This object performs no grading, producer sealing, archive/store write or + execution. H owns invocation, identity, read authority and failure handling. + """ + + def __init__(self, project: Callable, *, candidate_role: str, report_role: str): + if not callable(project) or any(not isinstance(role, str) or not role + for role in (candidate_role, report_role)): + raise ValueError("Expected a pure projector and nonempty configured roles.") + self._project = project + self._candidate_role = candidate_role + self._report_role = report_role + + def capture(self, phase, facts, *, artifacts): + if phase not in ("preparing", "finalizing"): + raise ValueError("Unknown evidence capture phase.") + saved = deepcopy(facts) + if saved["candidate"]["input_role"] != self._candidate_role: + raise PublicError("IdentityMismatch", stage="evidence_capture", + message="The actual candidate role differs from host configuration.") + candidate = _one([item["retained_ref"] for item in saved["bound_inputs"] + if item["role"] == self._candidate_role + and item["attempt_id"] == saved["attempt_id"]]) + report = None + if phase == "finalizing": + report = _one([item["artifact"] for step in saved["steps"] + for item in step["artifacts"] if item["role"] == self._report_role + and item["status"] == "available" + and item["attempt_id"] == saved["attempt_id"] + and item["step_id"] == step["step_id"]]) + return self._project( + phase, saved, candidate_bytes=_read(candidate, artifacts), + report_bytes=_read(report, artifacts), + ) diff --git a/tests/check_installed.py b/tests/check_installed.py index 295e8ca70..3bab5da79 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -16,7 +16,7 @@ import inferrsi import inferrsi.harness -from inferrsi.apps.api import CandidateProvider, EvaluationPort +from inferrsi.apps.api import CandidateProvider, EvaluationPort, EvidenceCapture from inferrsi.apps.cli import main from inferrsi.benchmark.registry import builtin_tasks, describe_task, task_resources from inferrsi.benchmark.skill_eval_engineering import REQUEST @@ -27,6 +27,8 @@ from inferrsi.resources import repository_packs from inferrsi.resources.repository_packs import REPO_TYPES, repository_pack +assert callable(EvidenceCapture) + # The new Harness binding port ships independently of Apps and Benchmark. subprocess.run([ sys.executable, "-I", "-B", "-c", diff --git a/tests/test_apps_capture_adapter.py b/tests/test_apps_capture_adapter.py new file mode 100644 index 000000000..cf5042866 --- /dev/null +++ b/tests/test_apps_capture_adapter.py @@ -0,0 +1,168 @@ +"""Role selection and byte authority in the thin Apps capture adapter.""" + +from copy import deepcopy +from hashlib import sha256 +from io import BytesIO + +import pytest + +from inferrsi.apps import api +from inferrsi.harness.errors import PublicError + + +@pytest.fixture +def supplied(): + data = {"candidate": b"actual candidate", "report": b"actual report"} + refs = {role: {"location": "retained/" + role, "sha256": sha256(raw).hexdigest(), + "size_bytes": len(raw), "media_type": "application/octet-stream"} + for role, raw in data.items()} + facts = {"attempt_id": "runtime-attempt", "candidate": {"input_role": "candidate"}, + "bound_inputs": [{"role": "candidate", "attempt_id": "runtime-attempt", + "retained_ref": refs["candidate"]}], + "steps": [{"step_id": "main", "artifacts": [{ + "role": "report", "attempt_id": "runtime-attempt", "step_id": "main", + "status": "available", "artifact": refs["report"], + }]}]} + calls, streams = [], [] + + class Reader: + def open(self, ref): + calls.append(deepcopy(ref)) + source = BytesIO(data[ref["location"].split("/")[-1]]) + streams.append(source) + return source + + return data, refs, facts, Reader(), calls, streams + + +@pytest.mark.parametrize("phase", ["preparing", "finalizing"]) +def test_roles_select_only_current_attempt_bytes_and_preserve_projector_result(supplied, phase): + data, refs, facts, reader, calls, streams = supplied + expected = object() + + def project(actual_phase, actual_facts, *, candidate_bytes, report_bytes): + assert actual_phase == phase and actual_facts == facts + assert candidate_bytes == data["candidate"] + assert report_bytes == (data["report"] if phase == "finalizing" else None) + actual_facts["candidate"]["input_role"] = "mutated" + return expected + + capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report") + assert capture.capture(phase, facts, artifacts=reader) is expected + assert calls == [refs["candidate"], *([refs["report"]] if phase == "finalizing" else [])] + assert facts["candidate"]["input_role"] == "candidate" + assert all(stream.closed for stream in streams) + + +@pytest.mark.parametrize("change", ["missing", "foreign-attempt", "foreign-step"]) +def test_absent_or_foreign_report_is_not_used_as_current_evidence(supplied, change): + _, _, facts, reader, _, _ = supplied + if change == "missing": + facts["steps"][0]["artifacts"].clear() + else: + key = "attempt_id" if change == "foreign-attempt" else "step_id" + facts["steps"][0]["artifacts"][0][key] = "foreign" + + def project(phase, actual_facts, *, candidate_bytes, report_bytes): + assert report_bytes is None and candidate_bytes is not None + return "unchanged result" + + capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report") + assert capture.capture("finalizing", facts, artifacts=reader) == "unchanged result" + + +@pytest.mark.parametrize("kind", ["candidate", "report"]) +def test_ambiguous_sources_are_not_resolved_by_first_or_last(supplied, kind): + _, _, facts, reader, _, _ = supplied + values = facts["bound_inputs"] if kind == "candidate" else facts["steps"][0]["artifacts"] + values.append(deepcopy(values[0])) + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("ambiguous capture reached B"), + candidate_role="candidate", report_role="report") + with pytest.raises(PublicError) as caught: + capture.capture("finalizing", facts, artifacts=reader) + assert caught.value.code == "IdentityMismatch" + + +def test_wrong_configured_candidate_role_refuses_before_any_byte_read(supplied): + _, _, facts, reader, calls, _ = supplied + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("foreign role reached B"), + candidate_role="other", report_role="report") + with pytest.raises(PublicError) as caught: + capture.capture("preparing", facts, artifacts=reader) + assert caught.value.code == "IdentityMismatch" and calls == [] + + +def test_reader_cannot_mutate_authoritative_ref_metadata(supplied): + data, _, facts, _, _, _ = supplied + before = deepcopy(facts) + + class MutatingReader: + def open(self, ref): + ref["sha256"] = "b" * 64 + return data["candidate"] + + capture = api.EvidenceCapture(lambda *a, **k: "unchanged", candidate_role="candidate", + report_role="report") + assert capture.capture("preparing", facts, artifacts=MutatingReader()) == "unchanged" + assert facts == before + + +def test_corrupt_capability_bytes_raise_integrity_error_and_close_stream(supplied): + data, _, facts, reader, _, streams = supplied + data["candidate"] = b"tampered" + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("unverified bytes reached B"), + candidate_role="candidate", report_role="report") + with pytest.raises(PublicError) as caught: + capture.capture("preparing", facts, artifacts=reader) + assert caught.value.code == "ArtifactIntegrityError" and streams[0].closed + + +def test_host_programming_errors_are_not_relabelled_as_evidence_gaps(supplied): + _, _, facts, _, _, _ = supplied + + class Broken: + def open(self, ref): + raise TypeError("synthetic host bug") + + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("broken host reached B"), + candidate_role="candidate", report_role="report") + with pytest.raises(TypeError, match="synthetic host bug"): + capture.capture("preparing", facts, artifacts=Broken()) + + +def test_short_binary_reads_are_collected_and_the_stream_is_closed(supplied): + data, _, facts, _, _, _ = supplied + + class Short(BytesIO): + def read(self, size): + return super().read(min(size, 2)) + + stream = Short(data["candidate"]) + + class Reader: + def open(self, ref): + return stream + + def project(phase, facts, *, candidate_bytes, report_bytes): + assert candidate_bytes == data["candidate"] and report_bytes is None + return "unchanged" + + capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report") + assert capture.capture("preparing", facts, artifacts=Reader()) == "unchanged" + assert stream.closed + + +@pytest.mark.parametrize("code", ["ArtifactUnavailable", "ArtifactIntegrityError", + "UnsafeArtifactPath"]) +def test_capability_refusals_keep_their_original_code_for_harness(supplied, code): + _, _, facts, _, _, _ = supplied + + class Refused: + def open(self, ref): + raise PublicError(code, stage="evidence_capture", message="synthetic refusal") + + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("refused source reached B"), + candidate_role="candidate", report_role="report") + with pytest.raises(PublicError) as caught: + capture.capture("preparing", facts, artifacts=Refused()) + assert caught.value.code == code diff --git a/tests/test_apps_evidence_capture.py b/tests/test_apps_evidence_capture.py new file mode 100644 index 000000000..08126d88a --- /dev/null +++ b/tests/test_apps_evidence_capture.py @@ -0,0 +1,223 @@ +"""Real Apps capture composition; diagnostic worker semantics, not task acceptance.""" + +import json +import sqlite3 +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path +from threading import Thread +from urllib.request import urlopen + +import pytest + +from inferrsi.apps import api +from inferrsi.apps.cli import main +from inferrsi.apps.storage import archive as archive_port +from inferrsi.apps.storage.archive import read_prepared +from inferrsi.apps.storage.artifact_reader import ArtifactReader +from inferrsi.apps.views.dashboard_live import DashboardServer +from inferrsi.benchmark import assessment, proofs +from inferrsi.benchmark import preparation as benchmark +from inferrsi.contracts.json_v1 import decode, encode +from inferrsi.harness import execution_service, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from tests.test_benchmark_preparation import preparation as preparation + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("candidate_bytes,worker_status,publish_failure", [ + (b"accept\n", "passed", False), (b"reject\n", "failed", False), + (b"reject\n", "failed", True), +]) +def test_actual_report_capture_and_assessment_share_runtime_refs( + preparation, tmp_path, monkeypatch, capsys, candidate_bytes, worker_status, publish_failure, +): + _, prepare_request, sources, protocol, _, config, archive = preparation + vector = decode((ROOT / "tests/fixtures/harness-prepared-binding/" + "d023-null-timeout.json").read_bytes()) + worker = b"""import json, pathlib, sys +candidate = pathlib.Path(sys.argv[1]).read_bytes() +status = 'passed' if candidate == b'accept\\n' else 'failed' +proof = {'schema_version': 1, 'grader': {'status': status, + 'reasons': [] if status == 'passed' else ['probe_rejection']}, 'samples': None} +output = pathlib.Path(sys.argv[2]) +output.parent.mkdir(parents=True, exist_ok=True) +output.write_text(json.dumps(proof), encoding='utf-8') +""" + sources["worker-code"].write_bytes(worker) + protocol = deepcopy(protocol) + protocol["worker_recipe"]["code_sha256"] = sha256(worker).hexdigest() + pin = next(x for x in protocol["resources"] if x["role"] == "worker-code") + pin["artifact"].update(sha256=sha256(worker).hexdigest(), size_bytes=len(worker)) + for required in protocol["required_evidence"]: + required["allowed_control_domains"] = [execution_service._CONTROL_DOMAIN] + service = benchmark.PreparationService( + protocol, resources=lambda role: sources[role].open("rb"), worker_role="worker-code", + ) + prepared_output = api.prepare( + {"schema_version": 1, "request": prepare_request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + output_id="probe-prepare", + ) + assert prepared_output.result["exit_code"] == 0 + prepared_ref = prepared_output.result["prepared_ref"] + prepared = read_prepared(config, prepared_ref) + vector.update(request_id="runtime-probe-47", attempt_id="runtime-probe-47", + run_id="runtime-run-47", output_root="attempts/runtime-probe-47") + vector["candidate"]["candidate_id"] = "runtime-candidate-47" + (archive / "candidate.txt").write_bytes(candidate_bytes) + candidate_ref = {"location": "archives/local/candidate.txt", "media_type": "text/plain", + "sha256": sha256(candidate_bytes).hexdigest(), + "size_bytes": len(candidate_bytes)} + proposal = {k: deepcopy(vector[k]) for k in ("candidate", "inputs", "generation_record_ref")} + proposal["inputs"][0].update(source=candidate_ref["location"], source_ref=candidate_ref, + expected_sha256=candidate_ref["sha256"]) + request = { + "schema_version": 1, "prepared_ref": prepared_ref, + "attempt": {key: vector[key] for key in execution_service.ATTEMPT_FIELDS}, + "proposal": proposal, "output_root": vector["output_root"], + } + modules = tmp_path / "modules" + modules.mkdir() + (modules / "fixture_worker.py").write_bytes(worker) + store = run_store.RunStore(tmp_path / "store", "test") + target = LocalCpuTarget("fixture-cpu", "1", work_root=tmp_path / "work", module_root=modules) + reader = ArtifactReader(config) + with reader.open(prepared["protocol"]) as stream: + projector = proofs.ProofProjector(prepared, stream.read()) + capture = api.EvidenceCapture( + projector.project, candidate_role=prepared["candidate_contract"]["input_role"], + report_role=prepared["recipe"]["outputs"][0]["role"], + ) + harness = Harness(store, target, artifact_reader=reader, evidence_capture=capture) + supervise = execution_service.local_process._supervise + pids = [] + + def counted(*args, **kwargs): + result = supervise(*args, **kwargs) + pids.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + def failed_publication(*args, **kwargs): + raise OSError("synthetic output publication failure") + + with monkeypatch.context() as publication: + if publish_failure: + publication.setattr(archive_port, "_publish_directory", failed_publication) + executed = api.execute(request, workspace=tmp_path, storage_config=config, + execute_harness=harness.execute, output_id="probe-execute") + assert executed.result["exit_code"] == (6 if publish_failure else 0), executed.result + record = executed.result["execution"] + record_ref = executed.result["record_ref"] + if publish_failure: + assert executed.result["error"]["latest_record_ref"] == record_ref + assert executed.result["reason_code"] == "ArchivePersistenceError" + assert record["started"] is True and record["process_outcome"] == "exited" + assert len(pids) == 1 and pids[0] is not None + assert record["cleanup"]["status"] == "confirmed" + assert record["target_binding"]["control_domain"] == execution_service._CONTROL_DOMAIN + runtime = record["target_binding"]["runtime_bindings_by_step"][0] + assert runtime["code_sha256"] == sha256(worker).hexdigest() + expected_checks = {"entry_code_identity", "input_identity", "format", "grader", "samples", + "cleanup"} + assert {x["check_id"] for x in record["checks"]} == expected_checks + output = next(x for x in record["artifacts"] if x["role"] == "grader") + assert output["status"] == "available" + with reader.open(output["artifact"]) as stream: + raw = stream.read() + proof = json.loads(raw) + assert proof["grader"]["status"] == worker_status + required = next(x for x in prepared["required_evidence"] if x["check_id"] == "grader") + check = next(x for x in record["checks"] if x["check_id"] == "grader") + assert check["producer"]["kind"] == "task_worker" + with reader.open(check["evidence_refs"][0]) as stream: + normalized = assessment._proof(stream.read(), required, record, candidate_ref["sha256"]) + assert normalized["attempt_id"] == "runtime-probe-47" + assert normalized["candidate_id"] == "runtime-candidate-47" + assert normalized["status"] == worker_status + + assessed = api.assess( + {"schema_version": 1, "record_ref": record_ref, + "primary_scope": "p1", "primary_claim_id": "C"}, workspace=tmp_path, + storage_config=config, assess_benchmark=assessment.assess, + validate_assessment=assessment.validate_assessment, encode_assessment=assessment.encode, + output_id="probe-assess", + ) + assert assessed.result["exit_code"] == (0 if worker_status == "passed" else 8), assessed.result + result = assessed.result["assessment"] + assert [(x["claim_id"], x["verdict"]) for x in result["claims"]] == [ + ("F", "PASS"), ("C", "PASS" if worker_status == "passed" else "FAIL"), + ("M", "INCONCLUSIVE"), + ] + assert result["claims"][2]["missing_evidence"] == ["samples"] + assert result["record_ref"] == record_ref + view = api.read_attempt_view( + record_ref, storage_config=config, assessment_ref=assessed.result["assessment_ref"], + include_trace=True, + ) + assert view["record_ref"] == record_ref + assert view["assessment"]["claims"] == [ + {k: x[k] for k in ("claim_id", "validation_scope", "verdict", "reasons", + "missing_evidence")} + for x in result["claims"] + ] + assert {x["check_id"] for x in view["trace"]["checks"]} == expected_checks + assert len(list(tmp_path.rglob("manifest.json"))) == (2 if publish_failure else 3) + assert len(list(tmp_path.rglob("result.json"))) == (2 if publish_failure else 3) + for output_id, expected in (("probe-prepare", prepared_output), ("probe-assess", assessed)): + assert archive_port.read_result(tmp_path, output_id).result == expected.result + if not publish_failure: + saved = archive_port.read_result(tmp_path, "probe-execute") + assert saved.result == executed.result and saved.result["record_ref"] == record_ref + + flags = [] + for name, value in (("record-ref", record_ref), ("storage-config", config), + ("assessment-ref", assessed.result["assessment_ref"])): + path = tmp_path / f"{name}.json" + path.write_bytes(encode(value)) + flags.extend((f"--{name}", str(path))) + assert main(["trace", *flags]) == 0 + assert json.loads(capsys.readouterr().out) == view + documents = {} + for command in ("report", "dashboard"): + html = tmp_path / f"{command}.html" + assert main([command, *flags, "--output", str(html)]) == 0 + capsys.readouterr() + document = html.read_text(encoding="utf-8") + assert record_ref["artifact"]["sha256"] in document + assert assessed.result["assessment_ref"]["artifact"]["sha256"] in document + assert "runtime-probe-47" in document and "INCONCLUSIVE" in document + documents[command] = document + with DashboardServer(None, None, 0, fixed_attempt=view) as server: + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + base = f"http://127.0.0.1:{server.server_port}" + with urlopen(f"{base}/api/state", timeout=5) as response: + assert json.load(response)["data"]["fixed_attempt"] == view + with urlopen(base, timeout=5) as response: + assert response.read().decode("utf-8") == documents["dashboard"] + finally: + server.shutdown() + thread.join(timeout=5) + + def forbidden(*args, **kwargs): + pytest.fail("completed attempt was implicitly executed again") + + class Never: + open = forbidden + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + replay = Harness(store, LocalCpuTarget("retired", "2", work_root=tmp_path / "retired"), + artifact_reader=Never(), evidence_capture=object()) + replayed = api.execute(request, workspace=tmp_path, storage_config=config, + execute_harness=replay.execute, output_id="probe-replay") + assert replayed.result["record_ref"] == record_ref + assert replayed.result["execution"]["request_digest"] == record["request_digest"] + assert archive_port.read_result(tmp_path, "probe-replay").result == replayed.result + with sqlite3.connect(store.root / run_store.INDEX_NAME) as conn: + assert conn.execute("SELECT count(*) FROM attempts").fetchone()[0] == 1 + assert conn.execute("SELECT count(*) FROM records").fetchone()[0] == 1