diff --git a/docs/contracts/harness-evidence-capture.md b/docs/contracts/harness-evidence-capture.md new file mode 100644 index 00000000..14cb5702 --- /dev/null +++ b/docs/contracts/harness-evidence-capture.md @@ -0,0 +1,91 @@ +# Harness evidence capture and producer sealing + +`Harness(..., evidence_capture=None)` accepts an optional trusted host capability +with `capture(phase, facts, *, artifacts) -> (draft_wire, proof_blobs)`. This is the +H slice accepted in [#565](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7063820381) +and its [stacking/failure clarification](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7064002232). +The capability is never named or instantiated by an ExecutionSpec or worker. +There are no Benchmark/Apps imports, task rules or new business wire fields in H. + +## Lifecycle + +Existing attempt lookup precedes all fresh preparation reads and capture calls. +Completed replay returns the original RecordRef; conflict/running/no-record +paths keep their existing errors without capture, execution or budget admission. +Ordinary mode never consults the capture configuration. `None` preserves the +existing evaluation behavior. + +For a fresh evaluation, H performs its existing admission, entry/input capture +and materialization. It then calls `preparing` before the worker starts. After +the one original run, output capture and cleanup/lease observations, it calls +`finalizing` before the single record commit. Capture consumes the original +attempt window; it creates no deadline, resource reservation, charge or retry. + +A valid failed or not_performed format observation is evidence rather than an +execution control: retain it and continue the one original worker under its +original limits. An illegal callback/configuration or identity/producer override +gives a registered zero-start fixed refusal. After a worker has run, rejected +capture results retain that actual execution/cleanup state and an explicit error; +they never restart it or turn it into a not_started record. + +## Snapshot and read authority + +`evidence_capture.CONTEXT_FIELDS` defines the closed v1 projection of existing H +facts: request/run/attempt/digest, preparation/protocol/operation/scope/candidate, +bound inputs, actual target/runtime binding, steps, started/outcome/exit code, +cleanup and leases. Preparing has no terminal observations (null fields). The +callback receives a defensive copy, not mutable spec/store/lease authority. + +The read capability authorizes only exact ArtifactRef metadata from this +attempt's retained sources at the current phase. It confines regular-file opens, +bounds reads to size+1, independently verifies size/hash, closes source streams, +and returns a separate binary stream. Ref aliases, arbitrary paths and other +attempts are refused. It expires after the callback. Host code is trusted; this +is a byte-authorization interface, not process isolation from malicious plugins. + +## Draft validation and sources + +Draft v1 has exactly `schema_version`, `checks`, `errors`. Each check has +`check_id`, `checker_revision`, `subject`, `phase`, `artifact_schema`, `status`, +`reasons`, nullable `proof_key`, and `source`. Errors have `code`, `check_ids` and +`reason`. Proof blobs are a separate map of identifier to immutable canonical +JSON bytes; there are no supplied Refs, paths or producer overrides. + +- `candidate_capture` is preparing-only. H requires the actual candidate source + to have been read through the capability and the proof's candidate/hash/attempt + to match its BoundInput. +- `worker_report` is finalizing-only, with a running/finalizing observation phase. + It additionally requires an actually started input-bound step, an observed + entry/revision/code pin matching the frozen step and a declared retained output + read through this capability. H seals task_worker identity from that runtime. +- `cleanup_capture` is finalizing-only. A supplied confirmation must agree with + H's cleanup/process-exit/acquired-lease facts. It cannot override unknown or + failed cleanup. + +H checks common identity and reserved producer fields without interpreting the +grader or sample rules. Supplied status must agree with a proof's status when +present. H alone seals producer kind/identity/revision, actual control domain and +capture route. A not_performed check without a proof is H's observation of the +gap; it does not impersonate a worker that produced no verified proof. + +## Retention and failures + +H computes proof hashes and tentative Refs from the snapshot bytes. Its existing +Claim.commit byte-source support retains all raw/derived artifacts before the +one durable record append. Valid RecordRefs are issued only after that commit. +Proof retention or append errors therefore use the existing +ExecutionPersistenceError with actual started state, partial facts and only +confirmed retained references. Uncommitted declarations are not published as +verified Refs. No Apps code writes a RunStore row. + +Well-formed draft errors preserve matching not_performed checks and any +independent valid families. Structurally invalid drafts have no trustworthy +task-check descriptor: H records its own `evaluation_capture` not_performed check +and a declared error, while retaining previous valid checks and raw output facts. +Unknown host programming exceptions propagate; the unfinished attempt remains +RecoveryRequired rather than being implicitly resumed. + +This supplies capture mechanics only. B's report codec/projector and the Apps +host adapter are separate. Real worker/grader/sampler semantics and resource +pins remain an explicit acceptance gate; synthetic observers and green smoke +tests do not establish complete F/C/M or whole-Issue production acceptance. diff --git a/docs/operations.md b/docs/operations.md index ac3ac527..cb4fefb2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -54,6 +54,7 @@ Effect classes used below: | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | | Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) | | Verify and execute one frozen source evaluation entry | Python `inferrsi.harness.api.Harness.execute` with trusted `LocalCpuTarget(module_root=...)` | Experiment-executing / record-writing | Fixed preparation and expected code digest; actual source captured after registration; T37 refusals start no worker; [source-entry contract](contracts/harness-entry-verification.md) | +| Capture and seal evaluation proofs | Python `Harness(..., evidence_capture=...)`; [capture contract](contracts/harness-evidence-capture.md) | Read-only capture callback / record-writing | H authorizes attempt bytes, seals actual provenance and retains drafts before one commit; ordinary mode and replay never call the observer | | Replay the same saved execution request | Python `inferrsi.harness.api.Harness.execute` | Existing-state query / fixed record-reading | Same attempt and canonical digest; existing state short-circuits fresh binding/execution; [replay and original limits](contracts/harness-evaluation-replay.md) | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | diff --git a/src/inferrsi/harness/api.py b/src/inferrsi/harness/api.py index 3643dc7f..523890c3 100644 --- a/src/inferrsi/harness/api.py +++ b/src/inferrsi/harness/api.py @@ -16,12 +16,14 @@ class Harness: """ def __init__( - self, store: run_store.RunStore, target: LocalCpuTarget, *, artifact_reader=None + self, store: run_store.RunStore, target: LocalCpuTarget, *, artifact_reader=None, + evidence_capture=None, ): self._store = store self._target = target # Optional reader with ``open(ArtifactRef)`` for artifact-namespace input sources. self._artifact_reader = artifact_reader + self._evidence_capture = evidence_capture def allocate( self, @@ -50,6 +52,7 @@ def execute(self, spec: Mapping, *, cancellation=None) -> dict: return execution_service.execute( self._store, self._target, spec, cancellation=cancellation, artifact_reader=self._artifact_reader, + evidence_capture=self._evidence_capture, ) def read(self, attempt_id: str, record_version: int | None = None) -> dict: diff --git a/src/inferrsi/harness/evidence_capture.py b/src/inferrsi/harness/evidence_capture.py new file mode 100644 index 00000000..1838148b --- /dev/null +++ b/src/inferrsi/harness/evidence_capture.py @@ -0,0 +1,288 @@ +"""H-owned attempt byte capability and pre-commit evidence provenance sealing.""" + +from __future__ import annotations + +from copy import deepcopy +from hashlib import sha256 +from io import BytesIO +from pathlib import Path + +from inferrsi.contracts import json_v1 +from inferrsi.harness import input_binding, run_store +from inferrsi.harness.errors import PublicError + +CONTEXT_FIELDS = frozenset( + "schema_version request_id run_id attempt_id request_digest prepared_ref protocol_digest " + "operation validation_scope candidate bound_inputs target_binding steps started " + "process_outcome exit_code cleanup resource_leases".split() +) +_CHECK = frozenset("check_id checker_revision subject phase artifact_schema status reasons " + "proof_key source".split()) +_ERROR_CODES = frozenset({ + "InvalidRequest", "InvalidExecutionSpec", "UnsupportedSchema", "UnsupportedRuleVersion", + "IdentityMismatch", "ArtifactUnavailable", "ArtifactIntegrityError", "UnsafeArtifactPath", +}) +_FATAL_CONFIG = {"InvalidExecutionSpec", "IdentityMismatch", "UnsupportedSchema", + "UnsupportedRuleVersion"} +_SOURCES = {"candidate_capture", "worker_report", "cleanup_capture"} + + +class _Rejected(ValueError): + def __init__(self, code="InvalidExecutionSpec"): + self.code = code + super().__init__("Invalid capture contract.") + + +def _require(value, code="InvalidExecutionSpec"): + if not value: + raise _Rejected(code) + + +def _snapshot(value): + try: + return json_v1.decode(json_v1.encode(value)) + except json_v1.CanonicalError as error: + raise _Rejected() from error + + +def _ids(values): + return isinstance(values, list) and all(run_store._is_id(value) for value in values) + + +class _Reader: + """Whitelist exact attempt Refs; return independently checked immutable bytes.""" + + def __init__(self, root, retained): + self.root = root + self.sources = {json_v1.encode(ref): source for ref, source in retained} + self.opened = set() + self.active = True + + def open(self, ref): + if not self.active: + raise PublicError("UnsafeArtifactPath", stage="evidence_capture", + message="The capture capability has expired.") + try: + key = json_v1.encode(ref) + except json_v1.CanonicalError: + key = None + if key not in self.sources: + raise PublicError("UnsafeArtifactPath", stage="evidence_capture", + message="Artifact is outside this attempt's retained sources.") + # Read the authorized metadata, not the caller's mutable object. + fixed = json_v1.decode(key) + source = self.sources[key] + try: + if isinstance(source, bytes): + data = source + else: + relative = Path(source).relative_to(self.root) + with input_binding.open_regular(self.root, relative.as_posix()) as stream: + chunks, remaining = [], fixed["size_bytes"] + 1 + while remaining: + chunk = stream.read(min(remaining, 1024 * 1024)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + except input_binding.BindingError as error: + raise PublicError("UnsafeArtifactPath", stage="evidence_capture", + message="Captured source is not a confined regular file.") from error + except (OSError, ValueError) as error: + raise PublicError("ArtifactUnavailable", stage="evidence_capture", + message="Captured source is unavailable.") from error + if (len(data) != fixed["size_bytes"] + or sha256(data).hexdigest() != fixed["sha256"]): + raise PublicError("ArtifactIntegrityError", stage="evidence_capture", + message="Captured source does not match its retained reference.") + self.opened.add(key) + return BytesIO(data) + + +def _read_source(reader, ref): + return isinstance(ref, dict) and json_v1.encode(ref) in reader.opened + + +def _candidate(facts): + candidate = facts["candidate"] + values = [item for item in facts["bound_inputs"] + if item["role"] == candidate["input_role"] + and item["attempt_id"] == facts["attempt_id"]] + _require(len(values) == 1, "IdentityMismatch") + return values[0] + + +def _cleanup_confirmed(facts): + cleanup, resources = facts["cleanup"], facts["resource_leases"] + return (isinstance(cleanup, dict) and cleanup.get("status") == "confirmed" + and cleanup.get("process_exit_confirmed") is True and isinstance(resources, dict) + and isinstance(resources.get("leases"), list) + and resources.get("applicability") in {"applicable", "not_applicable"} + and ((resources["applicability"] == "applicable" and bool(resources["leases"])) + or (resources["applicability"] == "not_applicable" and not resources["leases"])) + and all(item.get("release_confirmed") is True + and item.get("release_confirmed_at") is not None + for item in resources["leases"])) + + +def _worker(facts, spec, reader, bound): + _require(facts["started"] is True, "IdentityMismatch") + expected = {item["step_id"]: item for item in spec["steps"]} + runtimes = facts["target_binding"]["runtime_bindings_by_step"] + for step in facts["steps"]: + step_id = step["step_id"] + if step["started_at"] is None or step_id not in bound["step_ids"]: + continue + frozen = expected[step_id] + for runtime in runtimes: + if runtime["step_id"] != step_id: + continue + if not (runtime["code_sha256"] == frozen["expected_code_sha256"] + and all(runtime[key] == frozen[key] for key in + ("runtime_role", "entrypoint", "entrypoint_revision"))): + continue + roles = {item["role"] for item in frozen["outputs"]} + if any(item["role"] in roles and item["status"] == "available" + and _read_source(reader, item["artifact"]) for item in step["artifacts"]): + return runtime + raise _Rejected("IdentityMismatch") + + +def _harness_producer(domain, route): + return {"kind": "harness_capture", "identity": "inferrsi-harness", "revision": "1", + "control_domain": domain, "capture_route": route} + + +def _seal(check, raw, facts, spec, reader): + common = {"schema_version": 1, "attempt_id": facts["attempt_id"], + "operation": facts["operation"], "validation_scope": facts["validation_scope"]} + producer = _harness_producer(facts["target_binding"]["control_domain"], + "evaluation-capture-unavailable@1") + if raw is not None: + try: + proof = json_v1.decode(raw) + _require(json_v1.encode(proof) == raw) + except json_v1.CanonicalError as error: + raise _Rejected() from error + _require(isinstance(proof, dict)) + _require(type(proof.get("schema_version")) is int and proof["schema_version"] == 1, + "UnsupportedSchema") + _require(not set(proof) & {"producer", "control_domain", "capture_route", + "evidence_refs", "record_ref"}, "IdentityMismatch") + _require(all(proof.get(key) == value for key, value in common.items()), + "IdentityMismatch") + source = check["source"] + producer["capture_route"] = "evaluation-" + source + "@1" + if source == "cleanup_capture": + _require(type(proof.get("confirmed")) is bool + and proof["confirmed"] is bool(_cleanup_confirmed(facts)), + "IdentityMismatch") + else: + bound = _candidate(facts) + _require(_read_source(reader, bound["retained_ref"]) + and proof.get("candidate_id") == facts["candidate"]["candidate_id"] + and proof.get("actual_sha256") == bound["actual_sha256"], "IdentityMismatch") + if source == "worker_report": + runtime = _worker(facts, spec, reader, bound) + producer.update(kind="task_worker", identity=runtime["entrypoint"], + revision=runtime["entrypoint_revision"]) + if "status" in proof: + _require(proof["status"] == check["status"], "IdentityMismatch") + return { + **{key: deepcopy(check[key]) for key in + ("check_id", "checker_revision", "subject", "phase", "artifact_schema", + "status", "reasons")}, + **{key: value for key, value in common.items() if key != "schema_version"}, + "producer": producer, "evidence_refs": [], + } + + +def _validate_result(result, phase): + _require(isinstance(result, tuple) and len(result) == 2) + draft, blobs = result + draft = _snapshot(draft) + _require(isinstance(draft, dict) and draft.keys() == {"schema_version", "checks", "errors"}) + _require(type(draft["schema_version"]) is int and draft["schema_version"] == 1, + "UnsupportedSchema") + _require(isinstance(draft["checks"], list) and isinstance(draft["errors"], list) + and isinstance(blobs, dict) and all(run_store._is_id(key) for key in blobs)) + blobs = dict(blobs) + _require(all(isinstance(value, bytes) for value in blobs.values())) + used, ids = set(), set() + for check in draft["checks"]: + _require(isinstance(check, dict) and check.keys() == _CHECK) + _require(all(run_store._is_id(check[key]) for key in + ("check_id", "checker_revision", "subject")) and _ids(check["reasons"])) + definition = check["artifact_schema"] + _require(isinstance(definition, dict) and definition.keys() == {"name", "revision"} + and all(run_store._is_id(value) for value in definition.values())) + _require(isinstance(check["source"], str) and check["source"] in _SOURCES + and check["status"] in + ("passed", "failed", "not_performed")) + if phase == "preparing": + _require(check["source"] == "candidate_capture" and check["phase"] == "preparing") + else: + _require((check["source"] == "worker_report" + and check["phase"] in ("running", "finalizing")) + or (check["source"] == "cleanup_capture" and check["phase"] == "finalizing")) + key = check["proof_key"] + if check["status"] == "not_performed": + _require(key is None) + else: + _require(isinstance(key, str) and key in blobs) + used.add(key) + ids.add(check["check_id"]) + _require(used == set(blobs)) + for error in draft["errors"]: + _require(isinstance(error, dict) and error.keys() == {"code", "check_ids", "reason"} + and isinstance(error["code"], str) and error["code"] in _ERROR_CODES + and _ids(error["check_ids"]) + and bool(error["check_ids"]) and set(error["check_ids"]) <= ids + and run_store._is_id(error["reason"])) + return draft, blobs + + +def _rejection(phase, facts, code): + return ([{ + "check_id": "evaluation_capture", "checker_revision": "1", "subject": "evaluation", + "phase": phase, "artifact_schema": {"name": "capture-error", "revision": "1"}, + "status": "not_performed", "reasons": ["capture_contract_rejected"], + **{key: facts[key] for key in ("attempt_id", "operation", "validation_scope")}, + "producer": _harness_producer(facts["target_binding"]["control_domain"], + "evaluation-capture-rejected@1"), "evidence_refs": [], + }], [{"code": code, "stage": phase, "message": "capture_contract_rejected", + "retryable": False}], [], True) + + +def observe(capability, phase, facts, *, spec, root, namespace, retained): + """Return checks/errors/tentative byte sources/refusal; H commits all sources once.""" + facts, spec = _snapshot(facts), _snapshot(spec) + reader = _Reader(root, retained) + try: + callback = getattr(capability, "capture", None) + _require(callable(callback)) + result = callback(phase, deepcopy(facts), artifacts=reader) + draft, blobs = _validate_result(result, phase) + checks, sources = [], [] + for item in draft["checks"]: + raw = blobs.get(item["proof_key"]) + check = _seal(item, raw, facts, spec, reader) + if raw is not None: + ref = {"location": run_store.object_location(namespace, sha256(raw).hexdigest()), + "sha256": sha256(raw).hexdigest(), "size_bytes": len(raw), + "media_type": "application/json"} + check["evidence_refs"] = [ref] + sources.append((ref, raw)) + checks.append(check) + errors = [{"code": item["code"], "stage": phase, "message": item["reason"], + "retryable": False} for item in draft["errors"]] + refused = phase == "preparing" and any(item["code"] in _FATAL_CONFIG for item in errors) + return checks, errors, sources, refused + except Exception as error: + code = getattr(error, "code", None) + if not isinstance(code, str) or code not in _ERROR_CODES: + raise + return _rejection(phase, facts, code) + finally: + reader.active = False diff --git a/src/inferrsi/harness/execution_service.py b/src/inferrsi/harness/execution_service.py index ed755a45..278a9ded 100644 --- a/src/inferrsi/harness/execution_service.py +++ b/src/inferrsi/harness/execution_service.py @@ -12,7 +12,16 @@ from pathlib import Path from inferrsi.contracts import json_v1 -from inferrsi.harness import entry_code, input_binding, local_process, prepared_binding, run_store +from inferrsi.harness import ( + entry_code, + input_binding, + local_process, + prepared_binding, + run_store, +) +from inferrsi.harness import ( + evidence_capture as capture_service, +) from inferrsi.harness.errors import PublicError # The ordinary CPU path preserves V05 null mode. Evaluation requires an explicit trusted @@ -952,7 +961,30 @@ def _refusal(ctx, digest, binding, admission, bound, checks, errors, reason) -> ) -def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: +def _capture_evidence(ctx, capability, phase, digest, binding, bound, payload=None): + if payload is None: + facts = { + "schema_version": 1, + **{key: ctx.spec[key] for key in + ("request_id", "run_id", "attempt_id", "prepared_ref", "protocol_digest", + "operation", "validation_scope", "candidate")}, + "request_digest": digest, "target_binding": binding, "bound_inputs": bound, + "steps": [], "started": False, "process_outcome": None, "exit_code": None, + "cleanup": None, "resource_leases": None, + } + else: + facts = {key: payload[key] for key in capture_service.CONTEXT_FIELDS} + checks, errors, sources, refused = capture_service.observe( + capability, phase, facts, spec=ctx.spec, root=ctx.captures, + namespace=ctx.namespace, retained=ctx.retained, + ) + # Claim.commit already accepts byte sources and retains them before its one append. + # A retention/append failure therefore keeps the actual started/partial facts. + ctx.retained.extend(sources) + return checks, errors, refused + + +def _run_claimed(ctx, target, digest, reader, deadline, cancellation, evidence_capture) -> dict: capability_checks, errors = _capabilities(ctx.spec) admission, budget_errors = _budget(ctx.spec["budget"]) errors += budget_errors @@ -1021,6 +1053,15 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: except (OSError, input_binding.BindingError) as exc: raise _RetentionFailure("input materialization failed") from exc record["materialized_path"] = item["materialized_path"] + capture_errors = [] + if evaluation and evidence_capture is not None: + extra, capture_errors, refused = _capture_evidence( + ctx, evidence_capture, "preparing", digest, binding, bound, + ) + checks += extra + if refused: + return _refusal(ctx, digest, binding, admission, bound, checks, capture_errors, + "capture_refused") steps, runs = _run_steps(ctx, argvs, deadline, cancellation) outcome, exit_code = _outcome(steps) cleanup, leases = _cleanup(runs) @@ -1048,7 +1089,7 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: [_error(codes[outcome], "running", f"process outcome {outcome}")] if outcome in codes else [] ) - return _payload( + payload = _payload( ctx, request_digest=digest, target_binding=binding, bound_inputs=bound, # V18: finished only once every started process domain is confirmed closed. phase="finished" if cleanup["status"] in ("not_needed", "confirmed") @@ -1056,8 +1097,15 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: started=bool(launched), process_outcome=outcome, exit_code=exit_code, steps=steps, cancellation=_cancellation(ctx, runs), cleanup=cleanup, checks=checks, budget_admission=admission, usage=_usage(runs), resource_leases=leases, - errors=run_errors, + errors=capture_errors + run_errors, ) + if evaluation and evidence_capture is not None: + extra, errors, _ = _capture_evidence( + ctx, evidence_capture, "finalizing", digest, binding, bound, payload, + ) + payload["checks"] += extra + payload["errors"] += errors + return payload def execute( @@ -1067,6 +1115,7 @@ def execute( *, cancellation=None, artifact_reader=None, + evidence_capture=None, ) -> dict: """Execute one request or return its existing state before any new binding or start.""" evaluation = _validate_fields(spec) @@ -1099,7 +1148,8 @@ def execute( ctx.captures.mkdir(parents=True) except OSError as exc: raise _RetentionFailure("capture directory creation failed") from exc - payload = _run_claimed(ctx, target, digest, artifact_reader, deadline, cancellation) + payload = _run_claimed(ctx, target, digest, artifact_reader, deadline, cancellation, + evidence_capture) except _RetentionFailure as exc: # Raised only before any start: no start and no record (figure 6). shutil.rmtree(ctx.captures, ignore_errors=True) diff --git a/tests/check_installed.py b/tests/check_installed.py index f6da835c..7e403abf 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -32,6 +32,7 @@ sys.executable, "-I", "-B", "-c", "import sys; from inferrsi.harness import prepared_binding as p; " "from inferrsi.harness import entry_code; assert callable(entry_code.capture_module); " + "from inferrsi.harness import evidence_capture; assert callable(evidence_capture.observe); " "assert all(callable(getattr(p, n)) for n in ('read', 'project', 'verify')); " "assert not any(n == root or n.startswith(root + '.') for n in sys.modules " "for root in ('inferrsi.apps', 'inferrsi.benchmark', 'inferrsi.knowledge'))", diff --git a/tests/test_harness_evidence_capture.py b/tests/test_harness_evidence_capture.py new file mode 100644 index 00000000..6fb6b3cb --- /dev/null +++ b/tests/test_harness_evidence_capture.py @@ -0,0 +1,435 @@ +"""Real H capture lifecycle with a synthetic observer; no task-quality acceptance.""" + +import json +import sqlite3 +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from hashlib import sha256 + +import pytest + +from inferrsi.contracts.json_v1 import decode, encode +from inferrsi.harness import evidence_capture, execution_service, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from inferrsi.harness.errors import PublicError +from tests.test_benchmark_preparation import preparation as preparation +from tests.test_harness_entry_verification import evaluation as evaluation + +WORKER = b"""import json, pathlib, sys +data = pathlib.Path(sys.argv[1]).read_bytes() +target = pathlib.Path(sys.argv[2]) +target.parent.mkdir(parents=True, exist_ok=True) +target.write_text(json.dumps({'observed_size': len(data)}), encoding='utf-8') +""" + + +def check(key, source, phase, status, proof): + return { + "check_id": key, "checker_revision": "1", "subject": "candidate", "phase": phase, + "artifact_schema": {"name": key + "-proof", "revision": "1"}, + "status": status, "reasons": [] if status == "passed" else ["observer_result"], + "proof_key": key if proof is not None else None, "source": source, + } + + +class Observer: + """Test observer for transport/provenance, not a production B grader/projector.""" + + def __init__(self, *, format_status="passed", mutate=None): + self.format_status = format_status + self.mutate = mutate + self.calls = [] + self.readers = [] + + def capture(self, phase, facts, *, artifacts): + self.calls.append((phase, deepcopy(facts))) + self.readers.append(artifacts) + candidate = next(item for item in facts["bound_inputs"] + if item["role"] == facts["candidate"]["input_role"]) + with artifacts.open(candidate["retained_ref"]) as stream: + data = stream.read() + common = {"schema_version": 1, **{key: facts[key] for key in + ("attempt_id", "operation", "validation_scope")}} + identity = {**common, "candidate_id": facts["candidate"]["candidate_id"], + "actual_sha256": sha256(data).hexdigest()} + draft, blobs = {"schema_version": 1, "checks": [], "errors": []}, {} + if phase == "preparing": + proof = None if self.format_status == "not_performed" else { + **identity, "claim_id": "F", "checker_revision": "1", + "status": self.format_status, "reasons": [], + } + draft["checks"].append(check("format", "candidate_capture", "preparing", + self.format_status, proof)) + if proof is not None: + blobs["format"] = encode(proof) + else: + outputs = [item for step in facts["steps"] for item in step["artifacts"] + if item["role"] == "grader" and item["status"] == "available"] + proof = None + if outputs: + with artifacts.open(outputs[0]["artifact"]) as stream: + observed = json.loads(stream.read()) + assert observed["observed_size"] == len(data) + proof = {**identity, "claim_id": "C", "checker_revision": "1", + "status": "passed", "reasons": []} + blobs["grader"] = encode(proof) + draft["checks"].append(check("grader", "worker_report", "finalizing", + "passed" if proof is not None else "not_performed", + proof)) + confirmed = facts["cleanup"]["status"] == "confirmed" + blobs["cleanup"] = encode({**common, "confirmed": confirmed}) + draft["checks"].append(check("cleanup", "cleanup_capture", "finalizing", + "passed" if confirmed else "failed", blobs["cleanup"])) + if self.mutate: + self.mutate(phase, facts, artifacts, draft, blobs) + return draft, blobs + + +def host(case, observer): + case["spec"].update(attempt_id="runtime-capture-42", request_id="runtime-capture-42", + run_id="runtime-run-42", output_root="attempts/runtime-capture-42") + case["spec"]["candidate"]["candidate_id"] = "runtime-candidate-42" + return Harness(case["store"], case["target"], artifact_reader=case["reader"], + evidence_capture=observer) + + +def counts(case): + with sqlite3.connect(case["store"].root / run_store.INDEX_NAME) as connection: + return (connection.execute("SELECT count(*) FROM attempts").fetchone()[0], + connection.execute("SELECT count(*) FROM records").fetchone()[0]) + + +@pytest.mark.parametrize("evaluation", [{"worker_code": WORKER}], indirect=True) +class TestLifecycle: + def test_two_phases_retain_proofs_and_seal_actual_producers(self, evaluation, monkeypatch): + case, observer = evaluation, Observer() + service = host(case, observer) + supervise = execution_service.local_process._supervise + pids = [] + + def counted(*args, **kwargs): + result = supervise(*args, **kwargs) + pids.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + record = service.execute(case["spec"]) + assert record["started"] is True and record["exit_code"] == 0 + assert len(pids) == 1 and pids[0] is not None + assert [phase for phase, _ in observer.calls] == ["preparing", "finalizing"] + assert observer.calls[0][1]["started"] is False + assert observer.calls[0][1]["cleanup"] is None + checks = {item["check_id"]: item for item in record["checks"]} + for key in ("format", "grader", "cleanup"): + item = checks[key] + assert item["status"] == "passed" and item["attempt_id"] == "runtime-capture-42" + assert item["producer"]["control_domain"] == record["target_binding"]["control_domain"] + with case["reader"].open(item["evidence_refs"][0]) as stream: + proof = decode(stream.read()) + assert proof["attempt_id"] == "runtime-capture-42" + if key != "cleanup": + assert proof["candidate_id"] == "runtime-candidate-42" + assert checks["grader"]["producer"]["kind"] == "task_worker" + assert checks["grader"]["producer"]["identity"] == "fixture_worker" + assert checks["format"]["producer"]["kind"] == "harness_capture" + assert checks["cleanup"]["producer"]["kind"] == "harness_capture" + assert counts(case) == (1, 1) + assert case["store"].read("runtime-capture-42")["record_ref"] == record["record_ref"] + with pytest.raises(PublicError) as expired: + observer.readers[0].open(observer.calls[0][1]["bound_inputs"][0]["retained_ref"]) + assert expired.value.code == "UnsafeArtifactPath" + + @pytest.mark.parametrize("format_status", ["failed", "not_performed"]) + def test_valid_format_failure_keeps_the_one_original_run(self, evaluation, format_status): + observer = Observer(format_status=format_status) + record = host(evaluation, observer).execute(evaluation["spec"]) + assert record["started"] is True and record["exit_code"] == 0 + item = next(x for x in record["checks"] if x["check_id"] == "format") + assert item["status"] == format_status + assert bool(item["evidence_refs"]) is (format_status == "failed") + assert counts(evaluation) == (1, 1) + + @pytest.mark.parametrize("kind", ["producer", "proof-producer", "attempt", "schema", + "proof-version", "source", "orphan", + "status", "wrong-hash", "phase"]) + def test_illegal_preparing_callback_commits_zero_start_refusal( + self, evaluation, monkeypatch, kind, + ): + def mutate(phase, facts, reader, draft, blobs): + if phase != "preparing": + pytest.fail("fixed refusal reached finalizing capture") + if kind == "producer": + draft["checks"][0]["producer"] = {"kind": "harness_capture"} + elif kind == "schema": + draft["schema_version"] = 2 + elif kind == "source": + draft["checks"][0]["source"] = {} + elif kind == "orphan": + blobs["orphan"] = b"{}" + elif kind == "phase": + draft["checks"][0]["phase"] = "finalizing" + else: + proof = decode(blobs["format"]) + if kind == "attempt": + proof["attempt_id"] = "foreign" + elif kind == "wrong-hash": + proof["actual_sha256"] = "b" * 64 + elif kind == "proof-producer": + proof["producer"] = {"kind": "harness_capture"} + elif kind == "proof-version": + proof["schema_version"] = True + else: + proof["status"] = "failed" + blobs["format"] = encode(proof) + + def forbidden(*args, **kwargs): + pytest.fail("illegal capture launched a worker") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + observer = Observer(mutate=mutate) + record = host(evaluation, observer).execute(evaluation["spec"]) + assert record["started"] is False and record["process_outcome"] == "not_started" + assert all(step["status"] == "not_started" for step in record["steps"]) + assert record["resource_leases"]["leases"] == [] and counts(evaluation) == (1, 1) + assert record["checks"][-1]["check_id"] == "evaluation_capture" + assert record["checks"][-1]["status"] == "not_performed" + assert record["errors"] and len(observer.calls) == 1 + + @pytest.mark.parametrize("kind", ["identity", "cleanup"]) + def test_illegal_post_run_proof_keeps_execution_and_never_restarts(self, evaluation, kind): + def mutate(phase, facts, reader, draft, blobs): + if phase != "finalizing": + return + key = "grader" if kind == "identity" else "cleanup" + proof = decode(blobs[key]) + if kind == "identity": + proof["candidate_id"] = "foreign" + else: + proof["confirmed"] = not proof["confirmed"] + blobs[key] = encode(proof) + + observer = Observer(mutate=mutate) + service = host(evaluation, observer) + record = service.execute(evaluation["spec"]) + assert record["started"] is True and record["exit_code"] == 0 + assert record["cleanup"]["status"] == "confirmed" + assert record["checks"][-1]["check_id"] == "evaluation_capture" + assert record["errors"][-1]["code"] == "IdentityMismatch" + replay = service.execute(evaluation["spec"]) + assert replay["record_ref"] == record["record_ref"] and len(observer.calls) == 2 + assert counts(evaluation) == (1, 1) + + def test_callback_mutations_cannot_change_internal_identity_or_terminal_facts(self, evaluation): + def mutate(phase, facts, reader, draft, blobs): + facts["attempt_id"] = "foreign" + facts["candidate"]["candidate_id"] = "foreign" + facts["target_binding"]["control_domain"] = "foreign" + if facts["cleanup"] is not None: + facts["cleanup"]["status"] = "unknown" + + observer = Observer(mutate=mutate) + record = host(evaluation, observer).execute(evaluation["spec"]) + assert record["attempt_id"] == "runtime-capture-42" + assert record["candidate"]["candidate_id"] == "runtime-candidate-42" + assert record["cleanup"]["status"] == "confirmed" + assert record["target_binding"]["control_domain"] != "foreign" + assert record["errors"] == [] + + @pytest.mark.parametrize("failure", ["proof-retain", "append"]) + def test_post_run_persistence_failure_preserves_actual_started_and_no_implicit_resume( + self, evaluation, monkeypatch, failure, + ): + case, observer = evaluation, Observer() + service = host(case, observer) + if failure == "proof-retain": + retain = case["store"]._retain + + def fail_bytes(ref, source): + if isinstance(source, bytes): + raise OSError("synthetic proof retention failure") + return retain(ref, source) + + monkeypatch.setattr(case["store"], "_retain", fail_bytes) + else: + monkeypatch.setattr(case["store"], "_append", + lambda *a, **k: (_ for _ in ()).throw(OSError("synthetic"))) + with pytest.raises(PublicError) as failed: + service.execute(case["spec"]) + assert failed.value.code == "ExecutionPersistenceError" and failed.value.started is True + partial = failed.value.to_json()["partial_record"] + assert partial["started"] is True and partial["steps"][0]["exit_code"] == 0 + assert any(x["check_id"] == "grader" for x in partial["checks"]) + assert failed.value.latest_record_ref is None and counts(case) == (1, 0) + with pytest.raises(PublicError) as replayed: + service.execute(case["spec"]) + assert replayed.value.code == "RecoveryRequired" and len(observer.calls) == 2 + + def test_replay_and_digest_conflict_do_not_call_capture_or_start(self, evaluation, monkeypatch): + case, observer = evaluation, Observer() + service = host(case, observer) + record = service.execute(case["spec"]) + + def forbidden(*args, **kwargs): + pytest.fail("reentry called capture or launched a worker") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + retired = LocalCpuTarget("retired", "2", work_root=case["target"].work_root / "retired") + replay = Harness(case["store"], retired, artifact_reader=object(), + evidence_capture=object()) + assert replay.execute(case["spec"])["record_ref"] == record["record_ref"] + changed = deepcopy(case["spec"]) + changed["deadline_utc"] = "2098-01-01T00:00:00Z" + with pytest.raises(PublicError) as conflict: + replay.execute(changed) + assert conflict.value.code == "RequestConflict" and len(observer.calls) == 2 + + def test_unknown_host_programming_error_propagates_and_attempt_is_not_retried(self, evaluation): + def mutate(*args): + raise TypeError("synthetic host bug") + + observer = Observer(mutate=mutate) + service = host(evaluation, observer) + with pytest.raises(TypeError, match="synthetic host bug"): + service.execute(evaluation["spec"]) + assert counts(evaluation) == (1, 0) + with pytest.raises(PublicError) as replay: + service.execute(evaluation["spec"]) + assert replay.value.code == "RecoveryRequired" + + def test_report_version_gap_is_retained_without_erasing_independent_facts(self, evaluation): + def mutate(phase, facts, reader, draft, blobs): + if phase == "finalizing": + item = draft["checks"][0] + item.update(status="not_performed", proof_key=None, + reasons=["unsupported_report_revision"]) + blobs.pop("grader") + draft["errors"] = [{"code": "UnsupportedSchema", "check_ids": ["grader"], + "reason": "unsupported_report_revision"}] + + record = host(evaluation, Observer(mutate=mutate)).execute(evaluation["spec"]) + checks = {item["check_id"]: item for item in record["checks"]} + assert record["started"] is True and record["exit_code"] == 0 + assert checks["format"]["status"] == checks["cleanup"]["status"] == "passed" + assert checks["grader"]["status"] == "not_performed" + assert checks["grader"]["evidence_refs"] == [] + assert record["errors"][-1]["code"] == "UnsupportedSchema" + + def test_bad_capture_configuration_is_a_fixed_zero_start_refusal(self, evaluation, monkeypatch): + def forbidden(*args, **kwargs): + pytest.fail("bad capture configuration started a worker") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + record = host(evaluation, object()).execute(evaluation["spec"]) + assert record["started"] is False and record["errors"][-1]["code"] == "InvalidExecutionSpec" + assert counts(evaluation) == (1, 1) + + def test_unread_source_cannot_be_sealed_as_a_capture_proof(self, evaluation, monkeypatch): + class Unread: + def capture(self, phase, facts, *, artifacts): + proof = {"schema_version": 1, **{key: facts[key] for key in + ("attempt_id", "operation", "validation_scope")}, + "candidate_id": facts["candidate"]["candidate_id"], + "actual_sha256": facts["bound_inputs"][0]["actual_sha256"]} + return ({"schema_version": 1, "errors": [], "checks": [ + check("format", "candidate_capture", "preparing", "passed", proof), + ]}, {"format": encode(proof)}) + + def forbidden(*args, **kwargs): + pytest.fail("unobserved source assertion started a worker") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + record = host(evaluation, Unread()).execute(evaluation["spec"]) + assert record["started"] is False and record["errors"][-1]["code"] == "IdentityMismatch" + + def test_ordinary_mode_does_not_consult_capture_configuration(self, evaluation): + service = host(evaluation, object()) + spec = evaluation["spec"] + spec.update(prepared_ref=None, protocol_digest=None, operation=None, + validation_scope=None, candidate=None, inputs=[]) + spec["steps"][0].update(entrypoint_kind="executable", entrypoint=sys.executable, + expected_code_sha256=None, outputs=[], + args=["-c", "print('ordinary')"]) + record = service.execute(spec) + assert record["started"] is True and record["exit_code"] == 0 + assert record["checks"] == [] and record["prepared_ref"] is None + + def test_cancellation_seen_during_capture_never_starts_a_worker(self, evaluation, monkeypatch): + cancellation = threading.Event() + + def mutate(phase, *args): + if phase == "preparing": + cancellation.set() + + def forbidden(*args, **kwargs): + pytest.fail("capture reset the original cancellation state") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + observer = Observer(mutate=mutate) + record = host(evaluation, observer).execute(evaluation["spec"], cancellation=cancellation) + assert record["started"] is False and record["process_outcome"] == "cancelled" + assert record["resource_leases"]["leases"] == [] + assert len(observer.calls) == 2 and counts(evaluation) == (1, 1) + + +@pytest.mark.parametrize("evaluation", [{"worker_code": WORKER + b"import time\ntime.sleep(10)\n"}], + indirect=True) +def test_a_running_attempt_never_calls_capture_or_starts_a_second_worker(evaluation, monkeypatch): + case, observer, cancellation = evaluation, Observer(), threading.Event() + service = host(case, observer) + supervise = execution_service.local_process._supervise + calls, pids = [], [] + + def counted(*args, **kwargs): + calls.append(True) + result = supervise(*args, **kwargs) + pids.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + output = (case["target"].work_root / "workspaces" / case["spec"]["output_root"] + / "outputs/grader.json") + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(service.execute, case["spec"], cancellation=cancellation) + try: + end = time.monotonic() + 5 + while not output.is_file() and not future.done() and time.monotonic() < end: + time.sleep(0.01) + assert output.is_file() and not future.done() + with pytest.raises(PublicError) as replay: + service.execute(case["spec"]) + assert replay.value.code == "AttemptInProgress" + assert len(observer.calls) == 1 and len(calls) == 1 + finally: + cancellation.set() + record = future.result(timeout=10) + assert record["started"] is True and record["process_outcome"] == "cancelled" + assert record["cleanup"]["status"] == "confirmed" + assert len(pids) == 1 and pids[0] is not None + assert len(observer.calls) == 2 and counts(case) == (1, 1) + + +@pytest.mark.parametrize("failure", ["foreign", "tampered", "missing"]) +def test_reader_authorizes_exact_attempt_refs_and_checks_retained_bytes(tmp_path, failure): + data = b"actual capture" + path = tmp_path / "captured" + path.write_bytes(data) + ref = {"location": "stores/test/objects/" + sha256(data).hexdigest(), + "sha256": sha256(data).hexdigest(), "size_bytes": len(data), + "media_type": "application/octet-stream"} + reader = evidence_capture._Reader(tmp_path, [(ref, path)]) + requested = deepcopy(ref) + if failure == "foreign": + requested["location"] = "foreign/" + ref["sha256"] + elif failure == "tampered": + path.write_bytes(b"changed bytes") + else: + path.unlink() + with pytest.raises(PublicError) as caught: + reader.open(requested) + expected = {"foreign": "UnsafeArtifactPath", "tampered": "ArtifactIntegrityError", + "missing": "ArtifactUnavailable"} + assert caught.value.code == expected[failure] + assert reader.opened == set() diff --git a/tests/test_harness_execute.py b/tests/test_harness_execute.py index 94e01b96..86b85e2f 100644 --- a/tests/test_harness_execute.py +++ b/tests/test_harness_execute.py @@ -39,6 +39,7 @@ "inferrsi.harness.input_binding", "inferrsi.harness.local_process", "inferrsi.harness.prepared_binding", "inferrsi.harness.entry_code", + "inferrsi.harness.evidence_capture", "inferrsi.harness.atomic_json", "inferrsi.harness.run_store", "inferrsi.harness.errors", "inferrsi.harness.execution_spec", "inferrsi.contracts.json_v1",