From abc6de4ddd36a01c37187b64640e81015549bc50 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 23:11:52 +0800 Subject: [PATCH] feat(apps): assemble role-bound evidence capture ports Read only current attempt sources through the Harness capability and pass bytes to an injected pure projector. Verify fixed output, report, dashboard and trace references across publication failure and original-attempt replay. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/apps-module-migrations.json | 3 + docs/architecture.md | 1 + docs/contracts/apps-evidence-capture.md | 55 +++++ docs/module-layout.md | 5 + docs/operations.md | 1 + src/inferrsi/apps/api.py | 1 + src/inferrsi/apps/handoff/evidence_capture.py | 81 +++++++ tests/check_installed.py | 4 +- tests/test_apps_capture_adapter.py | 168 +++++++++++++ tests/test_apps_evidence_capture.py | 223 ++++++++++++++++++ 10 files changed, 541 insertions(+), 1 deletion(-) create mode 100644 docs/contracts/apps-evidence-capture.md create mode 100644 src/inferrsi/apps/handoff/evidence_capture.py create mode 100644 tests/test_apps_capture_adapter.py create mode 100644 tests/test_apps_evidence_capture.py diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index 73408ddc9..b3140f3de 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -30,6 +30,7 @@ "src/inferrsi/apps/workflows/benchmark_compare.py", "tests/check_installed.py", "tests/test_apps_assess.py", + "tests/test_apps_evidence_capture.py", "tests/test_apps_module_migrations.py", "tests/test_apps_output.py", "tests/test_apps_port_validation.py", @@ -61,6 +62,7 @@ "tests/check_installed.py", "tests/test_apps_artifact_reader.py", "tests/test_apps_assess.py", + "tests/test_apps_evidence_capture.py", "tests/test_apps_generation_plan.py", "tests/test_apps_module_migrations.py", "tests/test_benchmark_assessment.py", @@ -88,6 +90,7 @@ "src/inferrsi/apps/delivery/adapters.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", + "src/inferrsi/apps/handoff/evidence_capture.py", "src/inferrsi/apps/result.py", "src/inferrsi/apps/storage/archive.py", "src/inferrsi/apps/storage/artifact_reader.py", diff --git a/docs/architecture.md b/docs/architecture.md index d33f51d02..9420edb45 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -199,6 +199,7 @@ src/inferrsi/ bridge.py Bind preparations to specs and fixed records to evidence context_provider.py Deliver Knowledge bytes and pins for Harness sealing candidate_provider.py Plan generation and collect fixed candidate outputs + evidence_capture.py Read H-authorized bytes by role for an injected proof projector storage/ archive.py Immutable preparations/assessments and manifest.json/result.json artifact_reader.py Read verified artifacts through authorized mounts diff --git a/docs/contracts/apps-evidence-capture.md b/docs/contracts/apps-evidence-capture.md new file mode 100644 index 000000000..95bac57db --- /dev/null +++ b/docs/contracts/apps-evidence-capture.md @@ -0,0 +1,55 @@ +# Apps evidence capture assembly + +`inferrsi.apps.api.EvidenceCapture` is defined in +`apps/handoff/evidence_capture.py`. It is the thin host adapter accepted in +[#565](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7064458592). +It imports no Benchmark business module, performs no grading and writes no +archive or RunStore. It passes the injected projector's result through unchanged. + +## Host configuration + +After reading the fixed preparation and protocol through their existing archive +ports, the host constructs the supported B projector and configures role mapping: + +```python +from inferrsi.apps.api import EvidenceCapture +from inferrsi.benchmark.proofs import ProofProjector +from inferrsi.harness.api import Harness + +projector = ProofProjector(prepared, protocol_bytes) +capture = EvidenceCapture( + projector.project, + candidate_role=prepared["candidate_contract"]["input_role"], + report_role=prepared["recipe"]["outputs"][0]["role"], +) +harness = Harness(store, target, artifact_reader=reader, evidence_capture=capture) +``` + +The supported frozen P1 recipe has exactly one report output. Roles come from +validated frozen metadata; Apps defines no task name, report schema, predicate, +sampler or producer policy. The normal Apps execute port submits its saved +request to `harness.execute`. Existing callers can keep their configured ports. + +## Data and failure boundaries + +The callback snapshots H facts, selects this attempt's configured candidate +input, and reads only through the H capability. In finalizing it also selects an +available configured report belonging to this attempt and its enclosing step. +An absent family is passed as explicit `None`; multiple sources are refused as +IdentityMismatch instead of choosing the first or last. A configured candidate +role that differs from actual facts is also refused. + +Opened streams are closed, including failed reads. Reads are bounded to size+1 +and bytes are independently checked against fixed Ref size/hash. Capability +refusals keep their original portable code for H's lifecycle handling. Unknown +host/type errors propagate; they do not become fabricated report absence. + +The injected B projector receives only supplied bytes and a defensive context +snapshot. It owns proof construction and independent family outcomes. H validates +the returned drafts, retains raw/derived bytes and seals their actual provenance +before its record commit. Apps never adds a check to a committed record. + +Standard manifest/result publication, assessment and view/trace all retain the +same committed RecordRef. Output publication failure preserves that reference; +replay uses the original attempt and skips capture. These mechanics tests do not +accept a real task/grader/sampler specification or missing measurement evidence. diff --git a/docs/module-layout.md b/docs/module-layout.md index d84c22a44..01fbbc272 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -52,6 +52,11 @@ Knowledge bytes and pins for Harness sealing. `candidate_provider.py` plans generation and collects candidates from fixed Harness records. These modules consume `apps/storage/`; storage never imports back into handoff. +`evidence_capture.py` defines the new `apps.api.EvidenceCapture` host adapter. +It reads H-authorized candidate/report bytes by configured role and passes them +to an injected pure projector; H owns proof sealing and the one record commit. +There is no former flat-path alias for this new capability. + The three former Apps paths retain explicit public exports recorded in the [migration inventory](apps-module-migrations.json). `apps.api.CandidateProvider` and the former provider module export the one class defined in diff --git a/docs/operations.md b/docs/operations.md index 2c5809e77..d9d9a206b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -55,6 +55,7 @@ Effect classes used below: | Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) | | Verify and execute one frozen source evaluation entry | Python `inferrsi.harness.api.Harness.execute` with trusted `LocalCpuTarget(module_root=...)` | Experiment-executing / record-writing | Fixed preparation and expected code digest; actual source captured after registration; T37 refusals start no worker; [source-entry contract](contracts/harness-entry-verification.md) | | Capture and seal evaluation proofs | Python `Harness(..., evidence_capture=...)`; [capture contract](contracts/harness-evidence-capture.md) | Read-only capture callback / record-writing | H authorizes attempt bytes, seals actual provenance and retains drafts before one commit; ordinary mode and replay never call the observer | +| Assemble evidence capture by role | Python `apps.api.EvidenceCapture`; [host adapter](contracts/apps-evidence-capture.md) | Read-only byte assembly / injected pure projection | Reads only H-authorized candidate/report refs; no grading, producer sealing or record/archive writes; reentry is short-circuited by H | | Replay the same saved execution request | Python `inferrsi.harness.api.Harness.execute` | Existing-state query / fixed record-reading | Same attempt and canonical digest; existing state short-circuits fresh binding/execution; [replay and original limits](contracts/harness-evaluation-replay.md) | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Assess one fixed P1 record offline | Python `benchmark.assessment.assess`; [production assess](contracts/production-assess.md) | Read-only retained evidence / pure encoding | Verifies the frozen protocol, actual capture and proof bytes; returns independent claims without launching a worker or writing an archive; Apps selects the primary key and archives the result | diff --git a/src/inferrsi/apps/api.py b/src/inferrsi/apps/api.py index 0b137ee57..e466a2e3c 100644 --- a/src/inferrsi/apps/api.py +++ b/src/inferrsi/apps/api.py @@ -17,6 +17,7 @@ verify_protocol, ) from .handoff.candidate_provider import CandidateProvider as CandidateProvider +from .handoff.evidence_capture import EvidenceCapture as EvidenceCapture from .storage.archive import ( ResourceReader, RunOutput, diff --git a/src/inferrsi/apps/handoff/evidence_capture.py b/src/inferrsi/apps/handoff/evidence_capture.py new file mode 100644 index 000000000..fe1a6c9be --- /dev/null +++ b/src/inferrsi/apps/handoff/evidence_capture.py @@ -0,0 +1,81 @@ +"""Read attempt sources by configured role and invoke a host-supplied pure projector.""" + +from collections.abc import Callable +from contextlib import closing +from copy import deepcopy +from hashlib import sha256 + +from inferrsi.harness.errors import PublicError + +from ..storage.result import validate_artifact_ref + + +def _one(values): + if len(values) > 1: + raise PublicError("IdentityMismatch", stage="evidence_capture", + message="The configured capture role has multiple sources.") + return values[0] if values else None + + +def _read(ref, artifacts): + if ref is None: + return None + validate_artifact_ref(ref) + fixed = deepcopy(ref) + opened = artifacts.open(deepcopy(fixed)) + if isinstance(opened, bytes): + data = opened + else: + chunks, remaining = [], fixed["size_bytes"] + 1 + with closing(opened): + while remaining: + chunk = opened.read(min(remaining, 1024 * 1024)) + if not isinstance(chunk, bytes): + raise TypeError("A capture capability must supply binary bytes or streams.") + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + if len(data) != fixed["size_bytes"] or sha256(data).hexdigest() != fixed["sha256"]: + raise PublicError("ArtifactIntegrityError", stage="evidence_capture", + message="Capture bytes do not match their retained reference.") + return data + + +class EvidenceCapture: + """Apps host adapter; roles and pure projection are configured from frozen B metadata. + + This object performs no grading, producer sealing, archive/store write or + execution. H owns invocation, identity, read authority and failure handling. + """ + + def __init__(self, project: Callable, *, candidate_role: str, report_role: str): + if not callable(project) or any(not isinstance(role, str) or not role + for role in (candidate_role, report_role)): + raise ValueError("Expected a pure projector and nonempty configured roles.") + self._project = project + self._candidate_role = candidate_role + self._report_role = report_role + + def capture(self, phase, facts, *, artifacts): + if phase not in ("preparing", "finalizing"): + raise ValueError("Unknown evidence capture phase.") + saved = deepcopy(facts) + if saved["candidate"]["input_role"] != self._candidate_role: + raise PublicError("IdentityMismatch", stage="evidence_capture", + message="The actual candidate role differs from host configuration.") + candidate = _one([item["retained_ref"] for item in saved["bound_inputs"] + if item["role"] == self._candidate_role + and item["attempt_id"] == saved["attempt_id"]]) + report = None + if phase == "finalizing": + report = _one([item["artifact"] for step in saved["steps"] + for item in step["artifacts"] if item["role"] == self._report_role + and item["status"] == "available" + and item["attempt_id"] == saved["attempt_id"] + and item["step_id"] == step["step_id"]]) + return self._project( + phase, saved, candidate_bytes=_read(candidate, artifacts), + report_bytes=_read(report, artifacts), + ) diff --git a/tests/check_installed.py b/tests/check_installed.py index 295e8ca70..3bab5da79 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -16,7 +16,7 @@ import inferrsi import inferrsi.harness -from inferrsi.apps.api import CandidateProvider, EvaluationPort +from inferrsi.apps.api import CandidateProvider, EvaluationPort, EvidenceCapture from inferrsi.apps.cli import main from inferrsi.benchmark.registry import builtin_tasks, describe_task, task_resources from inferrsi.benchmark.skill_eval_engineering import REQUEST @@ -27,6 +27,8 @@ from inferrsi.resources import repository_packs from inferrsi.resources.repository_packs import REPO_TYPES, repository_pack +assert callable(EvidenceCapture) + # The new Harness binding port ships independently of Apps and Benchmark. subprocess.run([ sys.executable, "-I", "-B", "-c", diff --git a/tests/test_apps_capture_adapter.py b/tests/test_apps_capture_adapter.py new file mode 100644 index 000000000..cf5042866 --- /dev/null +++ b/tests/test_apps_capture_adapter.py @@ -0,0 +1,168 @@ +"""Role selection and byte authority in the thin Apps capture adapter.""" + +from copy import deepcopy +from hashlib import sha256 +from io import BytesIO + +import pytest + +from inferrsi.apps import api +from inferrsi.harness.errors import PublicError + + +@pytest.fixture +def supplied(): + data = {"candidate": b"actual candidate", "report": b"actual report"} + refs = {role: {"location": "retained/" + role, "sha256": sha256(raw).hexdigest(), + "size_bytes": len(raw), "media_type": "application/octet-stream"} + for role, raw in data.items()} + facts = {"attempt_id": "runtime-attempt", "candidate": {"input_role": "candidate"}, + "bound_inputs": [{"role": "candidate", "attempt_id": "runtime-attempt", + "retained_ref": refs["candidate"]}], + "steps": [{"step_id": "main", "artifacts": [{ + "role": "report", "attempt_id": "runtime-attempt", "step_id": "main", + "status": "available", "artifact": refs["report"], + }]}]} + calls, streams = [], [] + + class Reader: + def open(self, ref): + calls.append(deepcopy(ref)) + source = BytesIO(data[ref["location"].split("/")[-1]]) + streams.append(source) + return source + + return data, refs, facts, Reader(), calls, streams + + +@pytest.mark.parametrize("phase", ["preparing", "finalizing"]) +def test_roles_select_only_current_attempt_bytes_and_preserve_projector_result(supplied, phase): + data, refs, facts, reader, calls, streams = supplied + expected = object() + + def project(actual_phase, actual_facts, *, candidate_bytes, report_bytes): + assert actual_phase == phase and actual_facts == facts + assert candidate_bytes == data["candidate"] + assert report_bytes == (data["report"] if phase == "finalizing" else None) + actual_facts["candidate"]["input_role"] = "mutated" + return expected + + capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report") + assert capture.capture(phase, facts, artifacts=reader) is expected + assert calls == [refs["candidate"], *([refs["report"]] if phase == "finalizing" else [])] + assert facts["candidate"]["input_role"] == "candidate" + assert all(stream.closed for stream in streams) + + +@pytest.mark.parametrize("change", ["missing", "foreign-attempt", "foreign-step"]) +def test_absent_or_foreign_report_is_not_used_as_current_evidence(supplied, change): + _, _, facts, reader, _, _ = supplied + if change == "missing": + facts["steps"][0]["artifacts"].clear() + else: + key = "attempt_id" if change == "foreign-attempt" else "step_id" + facts["steps"][0]["artifacts"][0][key] = "foreign" + + def project(phase, actual_facts, *, candidate_bytes, report_bytes): + assert report_bytes is None and candidate_bytes is not None + return "unchanged result" + + capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report") + assert capture.capture("finalizing", facts, artifacts=reader) == "unchanged result" + + +@pytest.mark.parametrize("kind", ["candidate", "report"]) +def test_ambiguous_sources_are_not_resolved_by_first_or_last(supplied, kind): + _, _, facts, reader, _, _ = supplied + values = facts["bound_inputs"] if kind == "candidate" else facts["steps"][0]["artifacts"] + values.append(deepcopy(values[0])) + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("ambiguous capture reached B"), + candidate_role="candidate", report_role="report") + with pytest.raises(PublicError) as caught: + capture.capture("finalizing", facts, artifacts=reader) + assert caught.value.code == "IdentityMismatch" + + +def test_wrong_configured_candidate_role_refuses_before_any_byte_read(supplied): + _, _, facts, reader, calls, _ = supplied + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("foreign role reached B"), + candidate_role="other", report_role="report") + with pytest.raises(PublicError) as caught: + capture.capture("preparing", facts, artifacts=reader) + assert caught.value.code == "IdentityMismatch" and calls == [] + + +def test_reader_cannot_mutate_authoritative_ref_metadata(supplied): + data, _, facts, _, _, _ = supplied + before = deepcopy(facts) + + class MutatingReader: + def open(self, ref): + ref["sha256"] = "b" * 64 + return data["candidate"] + + capture = api.EvidenceCapture(lambda *a, **k: "unchanged", candidate_role="candidate", + report_role="report") + assert capture.capture("preparing", facts, artifacts=MutatingReader()) == "unchanged" + assert facts == before + + +def test_corrupt_capability_bytes_raise_integrity_error_and_close_stream(supplied): + data, _, facts, reader, _, streams = supplied + data["candidate"] = b"tampered" + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("unverified bytes reached B"), + candidate_role="candidate", report_role="report") + with pytest.raises(PublicError) as caught: + capture.capture("preparing", facts, artifacts=reader) + assert caught.value.code == "ArtifactIntegrityError" and streams[0].closed + + +def test_host_programming_errors_are_not_relabelled_as_evidence_gaps(supplied): + _, _, facts, _, _, _ = supplied + + class Broken: + def open(self, ref): + raise TypeError("synthetic host bug") + + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("broken host reached B"), + candidate_role="candidate", report_role="report") + with pytest.raises(TypeError, match="synthetic host bug"): + capture.capture("preparing", facts, artifacts=Broken()) + + +def test_short_binary_reads_are_collected_and_the_stream_is_closed(supplied): + data, _, facts, _, _, _ = supplied + + class Short(BytesIO): + def read(self, size): + return super().read(min(size, 2)) + + stream = Short(data["candidate"]) + + class Reader: + def open(self, ref): + return stream + + def project(phase, facts, *, candidate_bytes, report_bytes): + assert candidate_bytes == data["candidate"] and report_bytes is None + return "unchanged" + + capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report") + assert capture.capture("preparing", facts, artifacts=Reader()) == "unchanged" + assert stream.closed + + +@pytest.mark.parametrize("code", ["ArtifactUnavailable", "ArtifactIntegrityError", + "UnsafeArtifactPath"]) +def test_capability_refusals_keep_their_original_code_for_harness(supplied, code): + _, _, facts, _, _, _ = supplied + + class Refused: + def open(self, ref): + raise PublicError(code, stage="evidence_capture", message="synthetic refusal") + + capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("refused source reached B"), + candidate_role="candidate", report_role="report") + with pytest.raises(PublicError) as caught: + capture.capture("preparing", facts, artifacts=Refused()) + assert caught.value.code == code diff --git a/tests/test_apps_evidence_capture.py b/tests/test_apps_evidence_capture.py new file mode 100644 index 000000000..08126d88a --- /dev/null +++ b/tests/test_apps_evidence_capture.py @@ -0,0 +1,223 @@ +"""Real Apps capture composition; diagnostic worker semantics, not task acceptance.""" + +import json +import sqlite3 +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path +from threading import Thread +from urllib.request import urlopen + +import pytest + +from inferrsi.apps import api +from inferrsi.apps.cli import main +from inferrsi.apps.storage import archive as archive_port +from inferrsi.apps.storage.archive import read_prepared +from inferrsi.apps.storage.artifact_reader import ArtifactReader +from inferrsi.apps.views.dashboard_live import DashboardServer +from inferrsi.benchmark import assessment, proofs +from inferrsi.benchmark import preparation as benchmark +from inferrsi.contracts.json_v1 import decode, encode +from inferrsi.harness import execution_service, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from tests.test_benchmark_preparation import preparation as preparation + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("candidate_bytes,worker_status,publish_failure", [ + (b"accept\n", "passed", False), (b"reject\n", "failed", False), + (b"reject\n", "failed", True), +]) +def test_actual_report_capture_and_assessment_share_runtime_refs( + preparation, tmp_path, monkeypatch, capsys, candidate_bytes, worker_status, publish_failure, +): + _, prepare_request, sources, protocol, _, config, archive = preparation + vector = decode((ROOT / "tests/fixtures/harness-prepared-binding/" + "d023-null-timeout.json").read_bytes()) + worker = b"""import json, pathlib, sys +candidate = pathlib.Path(sys.argv[1]).read_bytes() +status = 'passed' if candidate == b'accept\\n' else 'failed' +proof = {'schema_version': 1, 'grader': {'status': status, + 'reasons': [] if status == 'passed' else ['probe_rejection']}, 'samples': None} +output = pathlib.Path(sys.argv[2]) +output.parent.mkdir(parents=True, exist_ok=True) +output.write_text(json.dumps(proof), encoding='utf-8') +""" + sources["worker-code"].write_bytes(worker) + protocol = deepcopy(protocol) + protocol["worker_recipe"]["code_sha256"] = sha256(worker).hexdigest() + pin = next(x for x in protocol["resources"] if x["role"] == "worker-code") + pin["artifact"].update(sha256=sha256(worker).hexdigest(), size_bytes=len(worker)) + for required in protocol["required_evidence"]: + required["allowed_control_domains"] = [execution_service._CONTROL_DOMAIN] + service = benchmark.PreparationService( + protocol, resources=lambda role: sources[role].open("rb"), worker_role="worker-code", + ) + prepared_output = api.prepare( + {"schema_version": 1, "request": prepare_request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + output_id="probe-prepare", + ) + assert prepared_output.result["exit_code"] == 0 + prepared_ref = prepared_output.result["prepared_ref"] + prepared = read_prepared(config, prepared_ref) + vector.update(request_id="runtime-probe-47", attempt_id="runtime-probe-47", + run_id="runtime-run-47", output_root="attempts/runtime-probe-47") + vector["candidate"]["candidate_id"] = "runtime-candidate-47" + (archive / "candidate.txt").write_bytes(candidate_bytes) + candidate_ref = {"location": "archives/local/candidate.txt", "media_type": "text/plain", + "sha256": sha256(candidate_bytes).hexdigest(), + "size_bytes": len(candidate_bytes)} + proposal = {k: deepcopy(vector[k]) for k in ("candidate", "inputs", "generation_record_ref")} + proposal["inputs"][0].update(source=candidate_ref["location"], source_ref=candidate_ref, + expected_sha256=candidate_ref["sha256"]) + request = { + "schema_version": 1, "prepared_ref": prepared_ref, + "attempt": {key: vector[key] for key in execution_service.ATTEMPT_FIELDS}, + "proposal": proposal, "output_root": vector["output_root"], + } + modules = tmp_path / "modules" + modules.mkdir() + (modules / "fixture_worker.py").write_bytes(worker) + store = run_store.RunStore(tmp_path / "store", "test") + target = LocalCpuTarget("fixture-cpu", "1", work_root=tmp_path / "work", module_root=modules) + reader = ArtifactReader(config) + with reader.open(prepared["protocol"]) as stream: + projector = proofs.ProofProjector(prepared, stream.read()) + capture = api.EvidenceCapture( + projector.project, candidate_role=prepared["candidate_contract"]["input_role"], + report_role=prepared["recipe"]["outputs"][0]["role"], + ) + harness = Harness(store, target, artifact_reader=reader, evidence_capture=capture) + supervise = execution_service.local_process._supervise + pids = [] + + def counted(*args, **kwargs): + result = supervise(*args, **kwargs) + pids.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + def failed_publication(*args, **kwargs): + raise OSError("synthetic output publication failure") + + with monkeypatch.context() as publication: + if publish_failure: + publication.setattr(archive_port, "_publish_directory", failed_publication) + executed = api.execute(request, workspace=tmp_path, storage_config=config, + execute_harness=harness.execute, output_id="probe-execute") + assert executed.result["exit_code"] == (6 if publish_failure else 0), executed.result + record = executed.result["execution"] + record_ref = executed.result["record_ref"] + if publish_failure: + assert executed.result["error"]["latest_record_ref"] == record_ref + assert executed.result["reason_code"] == "ArchivePersistenceError" + assert record["started"] is True and record["process_outcome"] == "exited" + assert len(pids) == 1 and pids[0] is not None + assert record["cleanup"]["status"] == "confirmed" + assert record["target_binding"]["control_domain"] == execution_service._CONTROL_DOMAIN + runtime = record["target_binding"]["runtime_bindings_by_step"][0] + assert runtime["code_sha256"] == sha256(worker).hexdigest() + expected_checks = {"entry_code_identity", "input_identity", "format", "grader", "samples", + "cleanup"} + assert {x["check_id"] for x in record["checks"]} == expected_checks + output = next(x for x in record["artifacts"] if x["role"] == "grader") + assert output["status"] == "available" + with reader.open(output["artifact"]) as stream: + raw = stream.read() + proof = json.loads(raw) + assert proof["grader"]["status"] == worker_status + required = next(x for x in prepared["required_evidence"] if x["check_id"] == "grader") + check = next(x for x in record["checks"] if x["check_id"] == "grader") + assert check["producer"]["kind"] == "task_worker" + with reader.open(check["evidence_refs"][0]) as stream: + normalized = assessment._proof(stream.read(), required, record, candidate_ref["sha256"]) + assert normalized["attempt_id"] == "runtime-probe-47" + assert normalized["candidate_id"] == "runtime-candidate-47" + assert normalized["status"] == worker_status + + assessed = api.assess( + {"schema_version": 1, "record_ref": record_ref, + "primary_scope": "p1", "primary_claim_id": "C"}, workspace=tmp_path, + storage_config=config, assess_benchmark=assessment.assess, + validate_assessment=assessment.validate_assessment, encode_assessment=assessment.encode, + output_id="probe-assess", + ) + assert assessed.result["exit_code"] == (0 if worker_status == "passed" else 8), assessed.result + result = assessed.result["assessment"] + assert [(x["claim_id"], x["verdict"]) for x in result["claims"]] == [ + ("F", "PASS"), ("C", "PASS" if worker_status == "passed" else "FAIL"), + ("M", "INCONCLUSIVE"), + ] + assert result["claims"][2]["missing_evidence"] == ["samples"] + assert result["record_ref"] == record_ref + view = api.read_attempt_view( + record_ref, storage_config=config, assessment_ref=assessed.result["assessment_ref"], + include_trace=True, + ) + assert view["record_ref"] == record_ref + assert view["assessment"]["claims"] == [ + {k: x[k] for k in ("claim_id", "validation_scope", "verdict", "reasons", + "missing_evidence")} + for x in result["claims"] + ] + assert {x["check_id"] for x in view["trace"]["checks"]} == expected_checks + assert len(list(tmp_path.rglob("manifest.json"))) == (2 if publish_failure else 3) + assert len(list(tmp_path.rglob("result.json"))) == (2 if publish_failure else 3) + for output_id, expected in (("probe-prepare", prepared_output), ("probe-assess", assessed)): + assert archive_port.read_result(tmp_path, output_id).result == expected.result + if not publish_failure: + saved = archive_port.read_result(tmp_path, "probe-execute") + assert saved.result == executed.result and saved.result["record_ref"] == record_ref + + flags = [] + for name, value in (("record-ref", record_ref), ("storage-config", config), + ("assessment-ref", assessed.result["assessment_ref"])): + path = tmp_path / f"{name}.json" + path.write_bytes(encode(value)) + flags.extend((f"--{name}", str(path))) + assert main(["trace", *flags]) == 0 + assert json.loads(capsys.readouterr().out) == view + documents = {} + for command in ("report", "dashboard"): + html = tmp_path / f"{command}.html" + assert main([command, *flags, "--output", str(html)]) == 0 + capsys.readouterr() + document = html.read_text(encoding="utf-8") + assert record_ref["artifact"]["sha256"] in document + assert assessed.result["assessment_ref"]["artifact"]["sha256"] in document + assert "runtime-probe-47" in document and "INCONCLUSIVE" in document + documents[command] = document + with DashboardServer(None, None, 0, fixed_attempt=view) as server: + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + base = f"http://127.0.0.1:{server.server_port}" + with urlopen(f"{base}/api/state", timeout=5) as response: + assert json.load(response)["data"]["fixed_attempt"] == view + with urlopen(base, timeout=5) as response: + assert response.read().decode("utf-8") == documents["dashboard"] + finally: + server.shutdown() + thread.join(timeout=5) + + def forbidden(*args, **kwargs): + pytest.fail("completed attempt was implicitly executed again") + + class Never: + open = forbidden + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + replay = Harness(store, LocalCpuTarget("retired", "2", work_root=tmp_path / "retired"), + artifact_reader=Never(), evidence_capture=object()) + replayed = api.execute(request, workspace=tmp_path, storage_config=config, + execute_harness=replay.execute, output_id="probe-replay") + assert replayed.result["record_ref"] == record_ref + assert replayed.result["execution"]["request_digest"] == record["request_digest"] + assert archive_port.read_result(tmp_path, "probe-replay").result == replayed.result + with sqlite3.connect(store.root / run_store.INDEX_NAME) as conn: + assert conn.execute("SELECT count(*) FROM attempts").fetchone()[0] == 1 + assert conn.execute("SELECT count(*) FROM records").fetchone()[0] == 1