From b5199316f659e3a10531d187223e8cf9db3720b5 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 22:43:39 +0800 Subject: [PATCH] feat(harness): seal evaluation capture proofs before commit Authorize only current attempt snapshots, bind observer drafts to actual execution facts and retain proof bytes with the existing single record commit. Replay and ordinary mode never call capture. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/harness-evidence-capture.md | 91 +++++ docs/operations.md | 1 + src/inferrsi/harness/api.py | 5 +- src/inferrsi/harness/evidence_capture.py | 288 ++++++++++++++ src/inferrsi/harness/execution_service.py | 60 ++- tests/check_installed.py | 1 + tests/test_harness_evidence_capture.py | 435 +++++++++++++++++++++ tests/test_harness_execute.py | 1 + 8 files changed, 876 insertions(+), 6 deletions(-) create mode 100644 docs/contracts/harness-evidence-capture.md create mode 100644 src/inferrsi/harness/evidence_capture.py create mode 100644 tests/test_harness_evidence_capture.py diff --git a/docs/contracts/harness-evidence-capture.md b/docs/contracts/harness-evidence-capture.md new file mode 100644 index 00000000..14cb5702 --- /dev/null +++ b/docs/contracts/harness-evidence-capture.md @@ -0,0 +1,91 @@ +# Harness evidence capture and producer sealing + +`Harness(..., evidence_capture=None)` accepts an optional trusted host capability +with `capture(phase, facts, *, artifacts) -> (draft_wire, proof_blobs)`. This is the +H slice accepted in [#565](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7063820381) +and its [stacking/failure clarification](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7064002232). +The capability is never named or instantiated by an ExecutionSpec or worker. +There are no Benchmark/Apps imports, task rules or new business wire fields in H. + +## Lifecycle + +Existing attempt lookup precedes all fresh preparation reads and capture calls. +Completed replay returns the original RecordRef; conflict/running/no-record +paths keep their existing errors without capture, execution or budget admission. +Ordinary mode never consults the capture configuration. `None` preserves the +existing evaluation behavior. + +For a fresh evaluation, H performs its existing admission, entry/input capture +and materialization. It then calls `preparing` before the worker starts. After +the one original run, output capture and cleanup/lease observations, it calls +`finalizing` before the single record commit. Capture consumes the original +attempt window; it creates no deadline, resource reservation, charge or retry. + +A valid failed or not_performed format observation is evidence rather than an +execution control: retain it and continue the one original worker under its +original limits. An illegal callback/configuration or identity/producer override +gives a registered zero-start fixed refusal. After a worker has run, rejected +capture results retain that actual execution/cleanup state and an explicit error; +they never restart it or turn it into a not_started record. + +## Snapshot and read authority + +`evidence_capture.CONTEXT_FIELDS` defines the closed v1 projection of existing H +facts: request/run/attempt/digest, preparation/protocol/operation/scope/candidate, +bound inputs, actual target/runtime binding, steps, started/outcome/exit code, +cleanup and leases. Preparing has no terminal observations (null fields). The +callback receives a defensive copy, not mutable spec/store/lease authority. + +The read capability authorizes only exact ArtifactRef metadata from this +attempt's retained sources at the current phase. It confines regular-file opens, +bounds reads to size+1, independently verifies size/hash, closes source streams, +and returns a separate binary stream. Ref aliases, arbitrary paths and other +attempts are refused. It expires after the callback. Host code is trusted; this +is a byte-authorization interface, not process isolation from malicious plugins. + +## Draft validation and sources + +Draft v1 has exactly `schema_version`, `checks`, `errors`. Each check has +`check_id`, `checker_revision`, `subject`, `phase`, `artifact_schema`, `status`, +`reasons`, nullable `proof_key`, and `source`. Errors have `code`, `check_ids` and +`reason`. Proof blobs are a separate map of identifier to immutable canonical +JSON bytes; there are no supplied Refs, paths or producer overrides. + +- `candidate_capture` is preparing-only. H requires the actual candidate source + to have been read through the capability and the proof's candidate/hash/attempt + to match its BoundInput. +- `worker_report` is finalizing-only, with a running/finalizing observation phase. + It additionally requires an actually started input-bound step, an observed + entry/revision/code pin matching the frozen step and a declared retained output + read through this capability. H seals task_worker identity from that runtime. +- `cleanup_capture` is finalizing-only. A supplied confirmation must agree with + H's cleanup/process-exit/acquired-lease facts. It cannot override unknown or + failed cleanup. + +H checks common identity and reserved producer fields without interpreting the +grader or sample rules. Supplied status must agree with a proof's status when +present. H alone seals producer kind/identity/revision, actual control domain and +capture route. A not_performed check without a proof is H's observation of the +gap; it does not impersonate a worker that produced no verified proof. + +## Retention and failures + +H computes proof hashes and tentative Refs from the snapshot bytes. Its existing +Claim.commit byte-source support retains all raw/derived artifacts before the +one durable record append. Valid RecordRefs are issued only after that commit. +Proof retention or append errors therefore use the existing +ExecutionPersistenceError with actual started state, partial facts and only +confirmed retained references. Uncommitted declarations are not published as +verified Refs. No Apps code writes a RunStore row. + +Well-formed draft errors preserve matching not_performed checks and any +independent valid families. Structurally invalid drafts have no trustworthy +task-check descriptor: H records its own `evaluation_capture` not_performed check +and a declared error, while retaining previous valid checks and raw output facts. +Unknown host programming exceptions propagate; the unfinished attempt remains +RecoveryRequired rather than being implicitly resumed. + +This supplies capture mechanics only. B's report codec/projector and the Apps +host adapter are separate. Real worker/grader/sampler semantics and resource +pins remain an explicit acceptance gate; synthetic observers and green smoke +tests do not establish complete F/C/M or whole-Issue production acceptance. diff --git a/docs/operations.md b/docs/operations.md index ac3ac527..cb4fefb2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -54,6 +54,7 @@ Effect classes used below: | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | | Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) | | Verify and execute one frozen source evaluation entry | Python `inferrsi.harness.api.Harness.execute` with trusted `LocalCpuTarget(module_root=...)` | Experiment-executing / record-writing | Fixed preparation and expected code digest; actual source captured after registration; T37 refusals start no worker; [source-entry contract](contracts/harness-entry-verification.md) | +| Capture and seal evaluation proofs | Python `Harness(..., evidence_capture=...)`; [capture contract](contracts/harness-evidence-capture.md) | Read-only capture callback / record-writing | H authorizes attempt bytes, seals actual provenance and retains drafts before one commit; ordinary mode and replay never call the observer | | Replay the same saved execution request | Python `inferrsi.harness.api.Harness.execute` | Existing-state query / fixed record-reading | Same attempt and canonical digest; existing state short-circuits fresh binding/execution; [replay and original limits](contracts/harness-evaluation-replay.md) | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | diff --git a/src/inferrsi/harness/api.py b/src/inferrsi/harness/api.py index 3643dc7f..523890c3 100644 --- a/src/inferrsi/harness/api.py +++ b/src/inferrsi/harness/api.py @@ -16,12 +16,14 @@ class Harness: """ def __init__( - self, store: run_store.RunStore, target: LocalCpuTarget, *, artifact_reader=None + self, store: run_store.RunStore, target: LocalCpuTarget, *, artifact_reader=None, + evidence_capture=None, ): self._store = store self._target = target # Optional reader with ``open(ArtifactRef)`` for artifact-namespace input sources. self._artifact_reader = artifact_reader + self._evidence_capture = evidence_capture def allocate( self, @@ -50,6 +52,7 @@ def execute(self, spec: Mapping, *, cancellation=None) -> dict: return execution_service.execute( self._store, self._target, spec, cancellation=cancellation, artifact_reader=self._artifact_reader, + evidence_capture=self._evidence_capture, ) def read(self, attempt_id: str, record_version: int | None = None) -> dict: diff --git a/src/inferrsi/harness/evidence_capture.py b/src/inferrsi/harness/evidence_capture.py new file mode 100644 index 00000000..1838148b --- /dev/null +++ b/src/inferrsi/harness/evidence_capture.py @@ -0,0 +1,288 @@ +"""H-owned attempt byte capability and pre-commit evidence provenance sealing.""" + +from __future__ import annotations + +from copy import deepcopy +from hashlib import sha256 +from io import BytesIO +from pathlib import Path + +from inferrsi.contracts import json_v1 +from inferrsi.harness import input_binding, run_store +from inferrsi.harness.errors import PublicError + +CONTEXT_FIELDS = frozenset( + "schema_version request_id run_id attempt_id request_digest prepared_ref protocol_digest " + "operation validation_scope candidate bound_inputs target_binding steps started " + "process_outcome exit_code cleanup resource_leases".split() +) +_CHECK = frozenset("check_id checker_revision subject phase artifact_schema status reasons " + "proof_key source".split()) +_ERROR_CODES = frozenset({ + "InvalidRequest", "InvalidExecutionSpec", "UnsupportedSchema", "UnsupportedRuleVersion", + "IdentityMismatch", "ArtifactUnavailable", "ArtifactIntegrityError", "UnsafeArtifactPath", +}) +_FATAL_CONFIG = {"InvalidExecutionSpec", "IdentityMismatch", "UnsupportedSchema", + "UnsupportedRuleVersion"} +_SOURCES = {"candidate_capture", "worker_report", "cleanup_capture"} + + +class _Rejected(ValueError): + def __init__(self, code="InvalidExecutionSpec"): + self.code = code + super().__init__("Invalid capture contract.") + + +def _require(value, code="InvalidExecutionSpec"): + if not value: + raise _Rejected(code) + + +def _snapshot(value): + try: + return json_v1.decode(json_v1.encode(value)) + except json_v1.CanonicalError as error: + raise _Rejected() from error + + +def _ids(values): + return isinstance(values, list) and all(run_store._is_id(value) for value in values) + + +class _Reader: + """Whitelist exact attempt Refs; return independently checked immutable bytes.""" + + def __init__(self, root, retained): + self.root = root + self.sources = {json_v1.encode(ref): source for ref, source in retained} + self.opened = set() + self.active = True + + def open(self, ref): + if not self.active: + raise PublicError("UnsafeArtifactPath", stage="evidence_capture", + message="The capture capability has expired.") + try: + key = json_v1.encode(ref) + except json_v1.CanonicalError: + key = None + if key not in self.sources: + raise PublicError("UnsafeArtifactPath", stage="evidence_capture", + message="Artifact is outside this attempt's retained sources.") + # Read the authorized metadata, not the caller's mutable object. + fixed = json_v1.decode(key) + source = self.sources[key] + try: + if isinstance(source, bytes): + data = source + else: + relative = Path(source).relative_to(self.root) + with input_binding.open_regular(self.root, relative.as_posix()) as stream: + chunks, remaining = [], fixed["size_bytes"] + 1 + while remaining: + chunk = stream.read(min(remaining, 1024 * 1024)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + except input_binding.BindingError as error: + raise PublicError("UnsafeArtifactPath", stage="evidence_capture", + message="Captured source is not a confined regular file.") from error + except (OSError, ValueError) as error: + raise PublicError("ArtifactUnavailable", stage="evidence_capture", + message="Captured source is unavailable.") from error + if (len(data) != fixed["size_bytes"] + or sha256(data).hexdigest() != fixed["sha256"]): + raise PublicError("ArtifactIntegrityError", stage="evidence_capture", + message="Captured source does not match its retained reference.") + self.opened.add(key) + return BytesIO(data) + + +def _read_source(reader, ref): + return isinstance(ref, dict) and json_v1.encode(ref) in reader.opened + + +def _candidate(facts): + candidate = facts["candidate"] + values = [item for item in facts["bound_inputs"] + if item["role"] == candidate["input_role"] + and item["attempt_id"] == facts["attempt_id"]] + _require(len(values) == 1, "IdentityMismatch") + return values[0] + + +def _cleanup_confirmed(facts): + cleanup, resources = facts["cleanup"], facts["resource_leases"] + return (isinstance(cleanup, dict) and cleanup.get("status") == "confirmed" + and cleanup.get("process_exit_confirmed") is True and isinstance(resources, dict) + and isinstance(resources.get("leases"), list) + and resources.get("applicability") in {"applicable", "not_applicable"} + and ((resources["applicability"] == "applicable" and bool(resources["leases"])) + or (resources["applicability"] == "not_applicable" and not resources["leases"])) + and all(item.get("release_confirmed") is True + and item.get("release_confirmed_at") is not None + for item in resources["leases"])) + + +def _worker(facts, spec, reader, bound): + _require(facts["started"] is True, "IdentityMismatch") + expected = {item["step_id"]: item for item in spec["steps"]} + runtimes = facts["target_binding"]["runtime_bindings_by_step"] + for step in facts["steps"]: + step_id = step["step_id"] + if step["started_at"] is None or step_id not in bound["step_ids"]: + continue + frozen = expected[step_id] + for runtime in runtimes: + if runtime["step_id"] != step_id: + continue + if not (runtime["code_sha256"] == frozen["expected_code_sha256"] + and all(runtime[key] == frozen[key] for key in + ("runtime_role", "entrypoint", "entrypoint_revision"))): + continue + roles = {item["role"] for item in frozen["outputs"]} + if any(item["role"] in roles and item["status"] == "available" + and _read_source(reader, item["artifact"]) for item in step["artifacts"]): + return runtime + raise _Rejected("IdentityMismatch") + + +def _harness_producer(domain, route): + return {"kind": "harness_capture", "identity": "inferrsi-harness", "revision": "1", + "control_domain": domain, "capture_route": route} + + +def _seal(check, raw, facts, spec, reader): + common = {"schema_version": 1, "attempt_id": facts["attempt_id"], + "operation": facts["operation"], "validation_scope": facts["validation_scope"]} + producer = _harness_producer(facts["target_binding"]["control_domain"], + "evaluation-capture-unavailable@1") + if raw is not None: + try: + proof = json_v1.decode(raw) + _require(json_v1.encode(proof) == raw) + except json_v1.CanonicalError as error: + raise _Rejected() from error + _require(isinstance(proof, dict)) + _require(type(proof.get("schema_version")) is int and proof["schema_version"] == 1, + "UnsupportedSchema") + _require(not set(proof) & {"producer", "control_domain", "capture_route", + "evidence_refs", "record_ref"}, "IdentityMismatch") + _require(all(proof.get(key) == value for key, value in common.items()), + "IdentityMismatch") + source = check["source"] + producer["capture_route"] = "evaluation-" + source + "@1" + if source == "cleanup_capture": + _require(type(proof.get("confirmed")) is bool + and proof["confirmed"] is bool(_cleanup_confirmed(facts)), + "IdentityMismatch") + else: + bound = _candidate(facts) + _require(_read_source(reader, bound["retained_ref"]) + and proof.get("candidate_id") == facts["candidate"]["candidate_id"] + and proof.get("actual_sha256") == bound["actual_sha256"], "IdentityMismatch") + if source == "worker_report": + runtime = _worker(facts, spec, reader, bound) + producer.update(kind="task_worker", identity=runtime["entrypoint"], + revision=runtime["entrypoint_revision"]) + if "status" in proof: + _require(proof["status"] == check["status"], "IdentityMismatch") + return { + **{key: deepcopy(check[key]) for key in + ("check_id", "checker_revision", "subject", "phase", "artifact_schema", + "status", "reasons")}, + **{key: value for key, value in common.items() if key != "schema_version"}, + "producer": producer, "evidence_refs": [], + } + + +def _validate_result(result, phase): + _require(isinstance(result, tuple) and len(result) == 2) + draft, blobs = result + draft = _snapshot(draft) + _require(isinstance(draft, dict) and draft.keys() == {"schema_version", "checks", "errors"}) + _require(type(draft["schema_version"]) is int and draft["schema_version"] == 1, + "UnsupportedSchema") + _require(isinstance(draft["checks"], list) and isinstance(draft["errors"], list) + and isinstance(blobs, dict) and all(run_store._is_id(key) for key in blobs)) + blobs = dict(blobs) + _require(all(isinstance(value, bytes) for value in blobs.values())) + used, ids = set(), set() + for check in draft["checks"]: + _require(isinstance(check, dict) and check.keys() == _CHECK) + _require(all(run_store._is_id(check[key]) for key in + ("check_id", "checker_revision", "subject")) and _ids(check["reasons"])) + definition = check["artifact_schema"] + _require(isinstance(definition, dict) and definition.keys() == {"name", "revision"} + and all(run_store._is_id(value) for value in definition.values())) + _require(isinstance(check["source"], str) and check["source"] in _SOURCES + and check["status"] in + ("passed", "failed", "not_performed")) + if phase == "preparing": + _require(check["source"] == "candidate_capture" and check["phase"] == "preparing") + else: + _require((check["source"] == "worker_report" + and check["phase"] in ("running", "finalizing")) + or (check["source"] == "cleanup_capture" and check["phase"] == "finalizing")) + key = check["proof_key"] + if check["status"] == "not_performed": + _require(key is None) + else: + _require(isinstance(key, str) and key in blobs) + used.add(key) + ids.add(check["check_id"]) + _require(used == set(blobs)) + for error in draft["errors"]: + _require(isinstance(error, dict) and error.keys() == {"code", "check_ids", "reason"} + and isinstance(error["code"], str) and error["code"] in _ERROR_CODES + and _ids(error["check_ids"]) + and bool(error["check_ids"]) and set(error["check_ids"]) <= ids + and run_store._is_id(error["reason"])) + return draft, blobs + + +def _rejection(phase, facts, code): + return ([{ + "check_id": "evaluation_capture", "checker_revision": "1", "subject": "evaluation", + "phase": phase, "artifact_schema": {"name": "capture-error", "revision": "1"}, + "status": "not_performed", "reasons": ["capture_contract_rejected"], + **{key: facts[key] for key in ("attempt_id", "operation", "validation_scope")}, + "producer": _harness_producer(facts["target_binding"]["control_domain"], + "evaluation-capture-rejected@1"), "evidence_refs": [], + }], [{"code": code, "stage": phase, "message": "capture_contract_rejected", + "retryable": False}], [], True) + + +def observe(capability, phase, facts, *, spec, root, namespace, retained): + """Return checks/errors/tentative byte sources/refusal; H commits all sources once.""" + facts, spec = _snapshot(facts), _snapshot(spec) + reader = _Reader(root, retained) + try: + callback = getattr(capability, "capture", None) + _require(callable(callback)) + result = callback(phase, deepcopy(facts), artifacts=reader) + draft, blobs = _validate_result(result, phase) + checks, sources = [], [] + for item in draft["checks"]: + raw = blobs.get(item["proof_key"]) + check = _seal(item, raw, facts, spec, reader) + if raw is not None: + ref = {"location": run_store.object_location(namespace, sha256(raw).hexdigest()), + "sha256": sha256(raw).hexdigest(), "size_bytes": len(raw), + "media_type": "application/json"} + check["evidence_refs"] = [ref] + sources.append((ref, raw)) + checks.append(check) + errors = [{"code": item["code"], "stage": phase, "message": item["reason"], + "retryable": False} for item in draft["errors"]] + refused = phase == "preparing" and any(item["code"] in _FATAL_CONFIG for item in errors) + return checks, errors, sources, refused + except Exception as error: + code = getattr(error, "code", None) + if not isinstance(code, str) or code not in _ERROR_CODES: + raise + return _rejection(phase, facts, code) + finally: + reader.active = False diff --git a/src/inferrsi/harness/execution_service.py b/src/inferrsi/harness/execution_service.py index ed755a45..278a9ded 100644 --- a/src/inferrsi/harness/execution_service.py +++ b/src/inferrsi/harness/execution_service.py @@ -12,7 +12,16 @@ from pathlib import Path from inferrsi.contracts import json_v1 -from inferrsi.harness import entry_code, input_binding, local_process, prepared_binding, run_store +from inferrsi.harness import ( + entry_code, + input_binding, + local_process, + prepared_binding, + run_store, +) +from inferrsi.harness import ( + evidence_capture as capture_service, +) from inferrsi.harness.errors import PublicError # The ordinary CPU path preserves V05 null mode. Evaluation requires an explicit trusted @@ -952,7 +961,30 @@ def _refusal(ctx, digest, binding, admission, bound, checks, errors, reason) -> ) -def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: +def _capture_evidence(ctx, capability, phase, digest, binding, bound, payload=None): + if payload is None: + facts = { + "schema_version": 1, + **{key: ctx.spec[key] for key in + ("request_id", "run_id", "attempt_id", "prepared_ref", "protocol_digest", + "operation", "validation_scope", "candidate")}, + "request_digest": digest, "target_binding": binding, "bound_inputs": bound, + "steps": [], "started": False, "process_outcome": None, "exit_code": None, + "cleanup": None, "resource_leases": None, + } + else: + facts = {key: payload[key] for key in capture_service.CONTEXT_FIELDS} + checks, errors, sources, refused = capture_service.observe( + capability, phase, facts, spec=ctx.spec, root=ctx.captures, + namespace=ctx.namespace, retained=ctx.retained, + ) + # Claim.commit already accepts byte sources and retains them before its one append. + # A retention/append failure therefore keeps the actual started/partial facts. + ctx.retained.extend(sources) + return checks, errors, refused + + +def _run_claimed(ctx, target, digest, reader, deadline, cancellation, evidence_capture) -> dict: capability_checks, errors = _capabilities(ctx.spec) admission, budget_errors = _budget(ctx.spec["budget"]) errors += budget_errors @@ -1021,6 +1053,15 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: except (OSError, input_binding.BindingError) as exc: raise _RetentionFailure("input materialization failed") from exc record["materialized_path"] = item["materialized_path"] + capture_errors = [] + if evaluation and evidence_capture is not None: + extra, capture_errors, refused = _capture_evidence( + ctx, evidence_capture, "preparing", digest, binding, bound, + ) + checks += extra + if refused: + return _refusal(ctx, digest, binding, admission, bound, checks, capture_errors, + "capture_refused") steps, runs = _run_steps(ctx, argvs, deadline, cancellation) outcome, exit_code = _outcome(steps) cleanup, leases = _cleanup(runs) @@ -1048,7 +1089,7 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: [_error(codes[outcome], "running", f"process outcome {outcome}")] if outcome in codes else [] ) - return _payload( + payload = _payload( ctx, request_digest=digest, target_binding=binding, bound_inputs=bound, # V18: finished only once every started process domain is confirmed closed. phase="finished" if cleanup["status"] in ("not_needed", "confirmed") @@ -1056,8 +1097,15 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: started=bool(launched), process_outcome=outcome, exit_code=exit_code, steps=steps, cancellation=_cancellation(ctx, runs), cleanup=cleanup, checks=checks, budget_admission=admission, usage=_usage(runs), resource_leases=leases, - errors=run_errors, + errors=capture_errors + run_errors, ) + if evaluation and evidence_capture is not None: + extra, errors, _ = _capture_evidence( + ctx, evidence_capture, "finalizing", digest, binding, bound, payload, + ) + payload["checks"] += extra + payload["errors"] += errors + return payload def execute( @@ -1067,6 +1115,7 @@ def execute( *, cancellation=None, artifact_reader=None, + evidence_capture=None, ) -> dict: """Execute one request or return its existing state before any new binding or start.""" evaluation = _validate_fields(spec) @@ -1099,7 +1148,8 @@ def execute( ctx.captures.mkdir(parents=True) except OSError as exc: raise _RetentionFailure("capture directory creation failed") from exc - payload = _run_claimed(ctx, target, digest, artifact_reader, deadline, cancellation) + payload = _run_claimed(ctx, target, digest, artifact_reader, deadline, cancellation, + evidence_capture) except _RetentionFailure as exc: # Raised only before any start: no start and no record (figure 6). shutil.rmtree(ctx.captures, ignore_errors=True) diff --git a/tests/check_installed.py b/tests/check_installed.py index f6da835c..7e403abf 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -32,6 +32,7 @@ sys.executable, "-I", "-B", "-c", "import sys; from inferrsi.harness import prepared_binding as p; " "from inferrsi.harness import entry_code; assert callable(entry_code.capture_module); " + "from inferrsi.harness import evidence_capture; assert callable(evidence_capture.observe); " "assert all(callable(getattr(p, n)) for n in ('read', 'project', 'verify')); " "assert not any(n == root or n.startswith(root + '.') for n in sys.modules " "for root in ('inferrsi.apps', 'inferrsi.benchmark', 'inferrsi.knowledge'))", diff --git a/tests/test_harness_evidence_capture.py b/tests/test_harness_evidence_capture.py new file mode 100644 index 00000000..6fb6b3cb --- /dev/null +++ b/tests/test_harness_evidence_capture.py @@ -0,0 +1,435 @@ +"""Real H capture lifecycle with a synthetic observer; no task-quality acceptance.""" + +import json +import sqlite3 +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from hashlib import sha256 + +import pytest + +from inferrsi.contracts.json_v1 import decode, encode +from inferrsi.harness import evidence_capture, execution_service, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from inferrsi.harness.errors import PublicError +from tests.test_benchmark_preparation import preparation as preparation +from tests.test_harness_entry_verification import evaluation as evaluation + +WORKER = b"""import json, pathlib, sys +data = pathlib.Path(sys.argv[1]).read_bytes() +target = pathlib.Path(sys.argv[2]) +target.parent.mkdir(parents=True, exist_ok=True) +target.write_text(json.dumps({'observed_size': len(data)}), encoding='utf-8') +""" + + +def check(key, source, phase, status, proof): + return { + "check_id": key, "checker_revision": "1", "subject": "candidate", "phase": phase, + "artifact_schema": {"name": key + "-proof", "revision": "1"}, + "status": status, "reasons": [] if status == "passed" else ["observer_result"], + "proof_key": key if proof is not None else None, "source": source, + } + + +class Observer: + """Test observer for transport/provenance, not a production B grader/projector.""" + + def __init__(self, *, format_status="passed", mutate=None): + self.format_status = format_status + self.mutate = mutate + self.calls = [] + self.readers = [] + + def capture(self, phase, facts, *, artifacts): + self.calls.append((phase, deepcopy(facts))) + self.readers.append(artifacts) + candidate = next(item for item in facts["bound_inputs"] + if item["role"] == facts["candidate"]["input_role"]) + with artifacts.open(candidate["retained_ref"]) as stream: + data = stream.read() + common = {"schema_version": 1, **{key: facts[key] for key in + ("attempt_id", "operation", "validation_scope")}} + identity = {**common, "candidate_id": facts["candidate"]["candidate_id"], + "actual_sha256": sha256(data).hexdigest()} + draft, blobs = {"schema_version": 1, "checks": [], "errors": []}, {} + if phase == "preparing": + proof = None if self.format_status == "not_performed" else { + **identity, "claim_id": "F", "checker_revision": "1", + "status": self.format_status, "reasons": [], + } + draft["checks"].append(check("format", "candidate_capture", "preparing", + self.format_status, proof)) + if proof is not None: + blobs["format"] = encode(proof) + else: + outputs = [item for step in facts["steps"] for item in step["artifacts"] + if item["role"] == "grader" and item["status"] == "available"] + proof = None + if outputs: + with artifacts.open(outputs[0]["artifact"]) as stream: + observed = json.loads(stream.read()) + assert observed["observed_size"] == len(data) + proof = {**identity, "claim_id": "C", "checker_revision": "1", + "status": "passed", "reasons": []} + blobs["grader"] = encode(proof) + draft["checks"].append(check("grader", "worker_report", "finalizing", + "passed" if proof is not None else "not_performed", + proof)) + confirmed = facts["cleanup"]["status"] == "confirmed" + blobs["cleanup"] = encode({**common, "confirmed": confirmed}) + draft["checks"].append(check("cleanup", "cleanup_capture", "finalizing", + "passed" if confirmed else "failed", blobs["cleanup"])) + if self.mutate: + self.mutate(phase, facts, artifacts, draft, blobs) + return draft, blobs + + +def host(case, observer): + case["spec"].update(attempt_id="runtime-capture-42", request_id="runtime-capture-42", + run_id="runtime-run-42", output_root="attempts/runtime-capture-42") + case["spec"]["candidate"]["candidate_id"] = "runtime-candidate-42" + return Harness(case["store"], case["target"], artifact_reader=case["reader"], + evidence_capture=observer) + + +def counts(case): + with sqlite3.connect(case["store"].root / run_store.INDEX_NAME) as connection: + return (connection.execute("SELECT count(*) FROM attempts").fetchone()[0], + connection.execute("SELECT count(*) FROM records").fetchone()[0]) + + +@pytest.mark.parametrize("evaluation", [{"worker_code": WORKER}], indirect=True) +class TestLifecycle: + def test_two_phases_retain_proofs_and_seal_actual_producers(self, evaluation, monkeypatch): + case, observer = evaluation, Observer() + service = host(case, observer) + supervise = execution_service.local_process._supervise + pids = [] + + def counted(*args, **kwargs): + result = supervise(*args, **kwargs) + pids.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + record = service.execute(case["spec"]) + assert record["started"] is True and record["exit_code"] == 0 + assert len(pids) == 1 and pids[0] is not None + assert [phase for phase, _ in observer.calls] == ["preparing", "finalizing"] + assert observer.calls[0][1]["started"] is False + assert observer.calls[0][1]["cleanup"] is None + checks = {item["check_id"]: item for item in record["checks"]} + for key in ("format", "grader", "cleanup"): + item = checks[key] + assert item["status"] == "passed" and item["attempt_id"] == "runtime-capture-42" + assert item["producer"]["control_domain"] == record["target_binding"]["control_domain"] + with case["reader"].open(item["evidence_refs"][0]) as stream: + proof = decode(stream.read()) + assert proof["attempt_id"] == "runtime-capture-42" + if key != "cleanup": + assert proof["candidate_id"] == "runtime-candidate-42" + assert checks["grader"]["producer"]["kind"] == "task_worker" + assert checks["grader"]["producer"]["identity"] == "fixture_worker" + assert checks["format"]["producer"]["kind"] == "harness_capture" + assert checks["cleanup"]["producer"]["kind"] == "harness_capture" + assert counts(case) == (1, 1) + assert case["store"].read("runtime-capture-42")["record_ref"] == record["record_ref"] + with pytest.raises(PublicError) as expired: + observer.readers[0].open(observer.calls[0][1]["bound_inputs"][0]["retained_ref"]) + assert expired.value.code == "UnsafeArtifactPath" + + @pytest.mark.parametrize("format_status", ["failed", "not_performed"]) + def test_valid_format_failure_keeps_the_one_original_run(self, evaluation, format_status): + observer = Observer(format_status=format_status) + record = host(evaluation, observer).execute(evaluation["spec"]) + assert record["started"] is True and record["exit_code"] == 0 + item = next(x for x in record["checks"] if x["check_id"] == "format") + assert item["status"] == format_status + assert bool(item["evidence_refs"]) is (format_status == "failed") + assert counts(evaluation) == (1, 1) + + @pytest.mark.parametrize("kind", ["producer", "proof-producer", "attempt", "schema", + "proof-version", "source", "orphan", + "status", "wrong-hash", "phase"]) + def test_illegal_preparing_callback_commits_zero_start_refusal( + self, evaluation, monkeypatch, kind, + ): + def mutate(phase, facts, reader, draft, blobs): + if phase != "preparing": + pytest.fail("fixed refusal reached finalizing capture") + if kind == "producer": + draft["checks"][0]["producer"] = {"kind": "harness_capture"} + elif kind == "schema": + draft["schema_version"] = 2 + elif kind == "source": + draft["checks"][0]["source"] = {} + elif kind == "orphan": + blobs["orphan"] = b"{}" + elif kind == "phase": + draft["checks"][0]["phase"] = "finalizing" + else: + proof = decode(blobs["format"]) + if kind == "attempt": + proof["attempt_id"] = "foreign" + elif kind == "wrong-hash": + proof["actual_sha256"] = "b" * 64 + elif kind == "proof-producer": + proof["producer"] = {"kind": "harness_capture"} + elif kind == "proof-version": + proof["schema_version"] = True + else: + proof["status"] = "failed" + blobs["format"] = encode(proof) + + def forbidden(*args, **kwargs): + pytest.fail("illegal capture launched a worker") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + observer = Observer(mutate=mutate) + record = host(evaluation, observer).execute(evaluation["spec"]) + assert record["started"] is False and record["process_outcome"] == "not_started" + assert all(step["status"] == "not_started" for step in record["steps"]) + assert record["resource_leases"]["leases"] == [] and counts(evaluation) == (1, 1) + assert record["checks"][-1]["check_id"] == "evaluation_capture" + assert record["checks"][-1]["status"] == "not_performed" + assert record["errors"] and len(observer.calls) == 1 + + @pytest.mark.parametrize("kind", ["identity", "cleanup"]) + def test_illegal_post_run_proof_keeps_execution_and_never_restarts(self, evaluation, kind): + def mutate(phase, facts, reader, draft, blobs): + if phase != "finalizing": + return + key = "grader" if kind == "identity" else "cleanup" + proof = decode(blobs[key]) + if kind == "identity": + proof["candidate_id"] = "foreign" + else: + proof["confirmed"] = not proof["confirmed"] + blobs[key] = encode(proof) + + observer = Observer(mutate=mutate) + service = host(evaluation, observer) + record = service.execute(evaluation["spec"]) + assert record["started"] is True and record["exit_code"] == 0 + assert record["cleanup"]["status"] == "confirmed" + assert record["checks"][-1]["check_id"] == "evaluation_capture" + assert record["errors"][-1]["code"] == "IdentityMismatch" + replay = service.execute(evaluation["spec"]) + assert replay["record_ref"] == record["record_ref"] and len(observer.calls) == 2 + assert counts(evaluation) == (1, 1) + + def test_callback_mutations_cannot_change_internal_identity_or_terminal_facts(self, evaluation): + def mutate(phase, facts, reader, draft, blobs): + facts["attempt_id"] = "foreign" + facts["candidate"]["candidate_id"] = "foreign" + facts["target_binding"]["control_domain"] = "foreign" + if facts["cleanup"] is not None: + facts["cleanup"]["status"] = "unknown" + + observer = Observer(mutate=mutate) + record = host(evaluation, observer).execute(evaluation["spec"]) + assert record["attempt_id"] == "runtime-capture-42" + assert record["candidate"]["candidate_id"] == "runtime-candidate-42" + assert record["cleanup"]["status"] == "confirmed" + assert record["target_binding"]["control_domain"] != "foreign" + assert record["errors"] == [] + + @pytest.mark.parametrize("failure", ["proof-retain", "append"]) + def test_post_run_persistence_failure_preserves_actual_started_and_no_implicit_resume( + self, evaluation, monkeypatch, failure, + ): + case, observer = evaluation, Observer() + service = host(case, observer) + if failure == "proof-retain": + retain = case["store"]._retain + + def fail_bytes(ref, source): + if isinstance(source, bytes): + raise OSError("synthetic proof retention failure") + return retain(ref, source) + + monkeypatch.setattr(case["store"], "_retain", fail_bytes) + else: + monkeypatch.setattr(case["store"], "_append", + lambda *a, **k: (_ for _ in ()).throw(OSError("synthetic"))) + with pytest.raises(PublicError) as failed: + service.execute(case["spec"]) + assert failed.value.code == "ExecutionPersistenceError" and failed.value.started is True + partial = failed.value.to_json()["partial_record"] + assert partial["started"] is True and partial["steps"][0]["exit_code"] == 0 + assert any(x["check_id"] == "grader" for x in partial["checks"]) + assert failed.value.latest_record_ref is None and counts(case) == (1, 0) + with pytest.raises(PublicError) as replayed: + service.execute(case["spec"]) + assert replayed.value.code == "RecoveryRequired" and len(observer.calls) == 2 + + def test_replay_and_digest_conflict_do_not_call_capture_or_start(self, evaluation, monkeypatch): + case, observer = evaluation, Observer() + service = host(case, observer) + record = service.execute(case["spec"]) + + def forbidden(*args, **kwargs): + pytest.fail("reentry called capture or launched a worker") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + retired = LocalCpuTarget("retired", "2", work_root=case["target"].work_root / "retired") + replay = Harness(case["store"], retired, artifact_reader=object(), + evidence_capture=object()) + assert replay.execute(case["spec"])["record_ref"] == record["record_ref"] + changed = deepcopy(case["spec"]) + changed["deadline_utc"] = "2098-01-01T00:00:00Z" + with pytest.raises(PublicError) as conflict: + replay.execute(changed) + assert conflict.value.code == "RequestConflict" and len(observer.calls) == 2 + + def test_unknown_host_programming_error_propagates_and_attempt_is_not_retried(self, evaluation): + def mutate(*args): + raise TypeError("synthetic host bug") + + observer = Observer(mutate=mutate) + service = host(evaluation, observer) + with pytest.raises(TypeError, match="synthetic host bug"): + service.execute(evaluation["spec"]) + assert counts(evaluation) == (1, 0) + with pytest.raises(PublicError) as replay: + service.execute(evaluation["spec"]) + assert replay.value.code == "RecoveryRequired" + + def test_report_version_gap_is_retained_without_erasing_independent_facts(self, evaluation): + def mutate(phase, facts, reader, draft, blobs): + if phase == "finalizing": + item = draft["checks"][0] + item.update(status="not_performed", proof_key=None, + reasons=["unsupported_report_revision"]) + blobs.pop("grader") + draft["errors"] = [{"code": "UnsupportedSchema", "check_ids": ["grader"], + "reason": "unsupported_report_revision"}] + + record = host(evaluation, Observer(mutate=mutate)).execute(evaluation["spec"]) + checks = {item["check_id"]: item for item in record["checks"]} + assert record["started"] is True and record["exit_code"] == 0 + assert checks["format"]["status"] == checks["cleanup"]["status"] == "passed" + assert checks["grader"]["status"] == "not_performed" + assert checks["grader"]["evidence_refs"] == [] + assert record["errors"][-1]["code"] == "UnsupportedSchema" + + def test_bad_capture_configuration_is_a_fixed_zero_start_refusal(self, evaluation, monkeypatch): + def forbidden(*args, **kwargs): + pytest.fail("bad capture configuration started a worker") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + record = host(evaluation, object()).execute(evaluation["spec"]) + assert record["started"] is False and record["errors"][-1]["code"] == "InvalidExecutionSpec" + assert counts(evaluation) == (1, 1) + + def test_unread_source_cannot_be_sealed_as_a_capture_proof(self, evaluation, monkeypatch): + class Unread: + def capture(self, phase, facts, *, artifacts): + proof = {"schema_version": 1, **{key: facts[key] for key in + ("attempt_id", "operation", "validation_scope")}, + "candidate_id": facts["candidate"]["candidate_id"], + "actual_sha256": facts["bound_inputs"][0]["actual_sha256"]} + return ({"schema_version": 1, "errors": [], "checks": [ + check("format", "candidate_capture", "preparing", "passed", proof), + ]}, {"format": encode(proof)}) + + def forbidden(*args, **kwargs): + pytest.fail("unobserved source assertion started a worker") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + record = host(evaluation, Unread()).execute(evaluation["spec"]) + assert record["started"] is False and record["errors"][-1]["code"] == "IdentityMismatch" + + def test_ordinary_mode_does_not_consult_capture_configuration(self, evaluation): + service = host(evaluation, object()) + spec = evaluation["spec"] + spec.update(prepared_ref=None, protocol_digest=None, operation=None, + validation_scope=None, candidate=None, inputs=[]) + spec["steps"][0].update(entrypoint_kind="executable", entrypoint=sys.executable, + expected_code_sha256=None, outputs=[], + args=["-c", "print('ordinary')"]) + record = service.execute(spec) + assert record["started"] is True and record["exit_code"] == 0 + assert record["checks"] == [] and record["prepared_ref"] is None + + def test_cancellation_seen_during_capture_never_starts_a_worker(self, evaluation, monkeypatch): + cancellation = threading.Event() + + def mutate(phase, *args): + if phase == "preparing": + cancellation.set() + + def forbidden(*args, **kwargs): + pytest.fail("capture reset the original cancellation state") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + observer = Observer(mutate=mutate) + record = host(evaluation, observer).execute(evaluation["spec"], cancellation=cancellation) + assert record["started"] is False and record["process_outcome"] == "cancelled" + assert record["resource_leases"]["leases"] == [] + assert len(observer.calls) == 2 and counts(evaluation) == (1, 1) + + +@pytest.mark.parametrize("evaluation", [{"worker_code": WORKER + b"import time\ntime.sleep(10)\n"}], + indirect=True) +def test_a_running_attempt_never_calls_capture_or_starts_a_second_worker(evaluation, monkeypatch): + case, observer, cancellation = evaluation, Observer(), threading.Event() + service = host(case, observer) + supervise = execution_service.local_process._supervise + calls, pids = [], [] + + def counted(*args, **kwargs): + calls.append(True) + result = supervise(*args, **kwargs) + pids.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + output = (case["target"].work_root / "workspaces" / case["spec"]["output_root"] + / "outputs/grader.json") + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(service.execute, case["spec"], cancellation=cancellation) + try: + end = time.monotonic() + 5 + while not output.is_file() and not future.done() and time.monotonic() < end: + time.sleep(0.01) + assert output.is_file() and not future.done() + with pytest.raises(PublicError) as replay: + service.execute(case["spec"]) + assert replay.value.code == "AttemptInProgress" + assert len(observer.calls) == 1 and len(calls) == 1 + finally: + cancellation.set() + record = future.result(timeout=10) + assert record["started"] is True and record["process_outcome"] == "cancelled" + assert record["cleanup"]["status"] == "confirmed" + assert len(pids) == 1 and pids[0] is not None + assert len(observer.calls) == 2 and counts(case) == (1, 1) + + +@pytest.mark.parametrize("failure", ["foreign", "tampered", "missing"]) +def test_reader_authorizes_exact_attempt_refs_and_checks_retained_bytes(tmp_path, failure): + data = b"actual capture" + path = tmp_path / "captured" + path.write_bytes(data) + ref = {"location": "stores/test/objects/" + sha256(data).hexdigest(), + "sha256": sha256(data).hexdigest(), "size_bytes": len(data), + "media_type": "application/octet-stream"} + reader = evidence_capture._Reader(tmp_path, [(ref, path)]) + requested = deepcopy(ref) + if failure == "foreign": + requested["location"] = "foreign/" + ref["sha256"] + elif failure == "tampered": + path.write_bytes(b"changed bytes") + else: + path.unlink() + with pytest.raises(PublicError) as caught: + reader.open(requested) + expected = {"foreign": "UnsafeArtifactPath", "tampered": "ArtifactIntegrityError", + "missing": "ArtifactUnavailable"} + assert caught.value.code == expected[failure] + assert reader.opened == set() diff --git a/tests/test_harness_execute.py b/tests/test_harness_execute.py index 94e01b96..86b85e2f 100644 --- a/tests/test_harness_execute.py +++ b/tests/test_harness_execute.py @@ -39,6 +39,7 @@ "inferrsi.harness.input_binding", "inferrsi.harness.local_process", "inferrsi.harness.prepared_binding", "inferrsi.harness.entry_code", + "inferrsi.harness.evidence_capture", "inferrsi.harness.atomic_json", "inferrsi.harness.run_store", "inferrsi.harness.errors", "inferrsi.harness.execution_spec", "inferrsi.contracts.json_v1",