Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/apps-module-migrations.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"src/inferrsi/apps/workflows/benchmark_compare.py",
"tests/check_installed.py",
"tests/test_apps_assess.py",
"tests/test_apps_evidence_capture.py",
"tests/test_apps_module_migrations.py",
"tests/test_apps_output.py",
"tests/test_apps_port_validation.py",
Expand Down Expand Up @@ -61,6 +62,7 @@
"tests/check_installed.py",
"tests/test_apps_artifact_reader.py",
"tests/test_apps_assess.py",
"tests/test_apps_evidence_capture.py",
"tests/test_apps_generation_plan.py",
"tests/test_apps_module_migrations.py",
"tests/test_benchmark_assessment.py",
Expand Down Expand Up @@ -88,6 +90,7 @@
"src/inferrsi/apps/delivery/adapters.py",
"src/inferrsi/apps/handoff/bridge.py",
"src/inferrsi/apps/handoff/candidate_provider.py",
"src/inferrsi/apps/handoff/evidence_capture.py",
"src/inferrsi/apps/result.py",
"src/inferrsi/apps/storage/archive.py",
"src/inferrsi/apps/storage/artifact_reader.py",
Expand Down
1 change: 1 addition & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ src/inferrsi/
bridge.py Bind preparations to specs and fixed records to evidence
context_provider.py Deliver Knowledge bytes and pins for Harness sealing
candidate_provider.py Plan generation and collect fixed candidate outputs
evidence_capture.py Read H-authorized bytes by role for an injected proof projector
storage/
archive.py Immutable preparations/assessments and manifest.json/result.json
artifact_reader.py Read verified artifacts through authorized mounts
Expand Down
55 changes: 55 additions & 0 deletions docs/contracts/apps-evidence-capture.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Apps evidence capture assembly

`inferrsi.apps.api.EvidenceCapture` is defined in
`apps/handoff/evidence_capture.py`. It is the thin host adapter accepted in
[#565](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7064458592).
It imports no Benchmark business module, performs no grading and writes no
archive or RunStore. It passes the injected projector's result through unchanged.

## Host configuration

After reading the fixed preparation and protocol through their existing archive
ports, the host constructs the supported B projector and configures role mapping:

```python
from inferrsi.apps.api import EvidenceCapture
from inferrsi.benchmark.proofs import ProofProjector
from inferrsi.harness.api import Harness

projector = ProofProjector(prepared, protocol_bytes)
capture = EvidenceCapture(
projector.project,
candidate_role=prepared["candidate_contract"]["input_role"],
report_role=prepared["recipe"]["outputs"][0]["role"],
)
harness = Harness(store, target, artifact_reader=reader, evidence_capture=capture)
```

The supported frozen P1 recipe has exactly one report output. Roles come from
validated frozen metadata; Apps defines no task name, report schema, predicate,
sampler or producer policy. The normal Apps execute port submits its saved
request to `harness.execute`. Existing callers can keep their configured ports.

## Data and failure boundaries

The callback snapshots H facts, selects this attempt's configured candidate
input, and reads only through the H capability. In finalizing it also selects an
available configured report belonging to this attempt and its enclosing step.
An absent family is passed as explicit `None`; multiple sources are refused as
IdentityMismatch instead of choosing the first or last. A configured candidate
role that differs from actual facts is also refused.

Opened streams are closed, including failed reads. Reads are bounded to size+1
and bytes are independently checked against fixed Ref size/hash. Capability
refusals keep their original portable code for H's lifecycle handling. Unknown
host/type errors propagate; they do not become fabricated report absence.

The injected B projector receives only supplied bytes and a defensive context
snapshot. It owns proof construction and independent family outcomes. H validates
the returned drafts, retains raw/derived bytes and seals their actual provenance
before its record commit. Apps never adds a check to a committed record.

Standard manifest/result publication, assessment and view/trace all retain the
same committed RecordRef. Output publication failure preserves that reference;
replay uses the original attempt and skips capture. These mechanics tests do not
accept a real task/grader/sampler specification or missing measurement evidence.
5 changes: 5 additions & 0 deletions docs/module-layout.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ Knowledge bytes and pins for Harness sealing. `candidate_provider.py` plans
generation and collects candidates from fixed Harness records. These modules
consume `apps/storage/`; storage never imports back into handoff.

`evidence_capture.py` defines the new `apps.api.EvidenceCapture` host adapter.
It reads H-authorized candidate/report bytes by configured role and passes them
to an injected pure projector; H owns proof sealing and the one record commit.
There is no former flat-path alias for this new capability.

The three former Apps paths retain explicit public exports recorded in the
[migration inventory](apps-module-migrations.json). `apps.api.CandidateProvider`
and the former provider module export the one class defined in
Expand Down
1 change: 1 addition & 0 deletions docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ Effect classes used below:
| Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) |
| Verify and execute one frozen source evaluation entry | Python `inferrsi.harness.api.Harness.execute` with trusted `LocalCpuTarget(module_root=...)` | Experiment-executing / record-writing | Fixed preparation and expected code digest; actual source captured after registration; T37 refusals start no worker; [source-entry contract](contracts/harness-entry-verification.md) |
| Capture and seal evaluation proofs | Python `Harness(..., evidence_capture=...)`; [capture contract](contracts/harness-evidence-capture.md) | Read-only capture callback / record-writing | H authorizes attempt bytes, seals actual provenance and retains drafts before one commit; ordinary mode and replay never call the observer |
| Assemble evidence capture by role | Python `apps.api.EvidenceCapture`; [host adapter](contracts/apps-evidence-capture.md) | Read-only byte assembly / injected pure projection | Reads only H-authorized candidate/report refs; no grading, producer sealing or record/archive writes; reentry is short-circuited by H |
| Replay the same saved execution request | Python `inferrsi.harness.api.Harness.execute` | Existing-state query / fixed record-reading | Same attempt and canonical digest; existing state short-circuits fresh binding/execution; [replay and original limits](contracts/harness-evaluation-replay.md) |
| Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref |
| Assess one fixed P1 record offline | Python `benchmark.assessment.assess`; [production assess](contracts/production-assess.md) | Read-only retained evidence / pure encoding | Verifies the frozen protocol, actual capture and proof bytes; returns independent claims without launching a worker or writing an archive; Apps selects the primary key and archives the result |
Expand Down
1 change: 1 addition & 0 deletions src/inferrsi/apps/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
verify_protocol,
)
from .handoff.candidate_provider import CandidateProvider as CandidateProvider
from .handoff.evidence_capture import EvidenceCapture as EvidenceCapture
from .storage.archive import (
ResourceReader,
RunOutput,
Expand Down
81 changes: 81 additions & 0 deletions src/inferrsi/apps/handoff/evidence_capture.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
"""Read attempt sources by configured role and invoke a host-supplied pure projector."""

from collections.abc import Callable
from contextlib import closing
from copy import deepcopy
from hashlib import sha256

from inferrsi.harness.errors import PublicError

from ..storage.result import validate_artifact_ref


def _one(values):
if len(values) > 1:
raise PublicError("IdentityMismatch", stage="evidence_capture",
message="The configured capture role has multiple sources.")
return values[0] if values else None


def _read(ref, artifacts):
if ref is None:
return None
validate_artifact_ref(ref)
fixed = deepcopy(ref)
opened = artifacts.open(deepcopy(fixed))
if isinstance(opened, bytes):
data = opened
else:
chunks, remaining = [], fixed["size_bytes"] + 1
with closing(opened):
while remaining:
chunk = opened.read(min(remaining, 1024 * 1024))
if not isinstance(chunk, bytes):
raise TypeError("A capture capability must supply binary bytes or streams.")
if not chunk:
break
chunks.append(chunk)
remaining -= len(chunk)
data = b"".join(chunks)
if len(data) != fixed["size_bytes"] or sha256(data).hexdigest() != fixed["sha256"]:
raise PublicError("ArtifactIntegrityError", stage="evidence_capture",
message="Capture bytes do not match their retained reference.")
return data


class EvidenceCapture:
"""Apps host adapter; roles and pure projection are configured from frozen B metadata.

This object performs no grading, producer sealing, archive/store write or
execution. H owns invocation, identity, read authority and failure handling.
"""

def __init__(self, project: Callable, *, candidate_role: str, report_role: str):
if not callable(project) or any(not isinstance(role, str) or not role
for role in (candidate_role, report_role)):
raise ValueError("Expected a pure projector and nonempty configured roles.")
self._project = project
self._candidate_role = candidate_role
self._report_role = report_role

def capture(self, phase, facts, *, artifacts):
if phase not in ("preparing", "finalizing"):
raise ValueError("Unknown evidence capture phase.")
saved = deepcopy(facts)
if saved["candidate"]["input_role"] != self._candidate_role:
raise PublicError("IdentityMismatch", stage="evidence_capture",
message="The actual candidate role differs from host configuration.")
candidate = _one([item["retained_ref"] for item in saved["bound_inputs"]
if item["role"] == self._candidate_role
and item["attempt_id"] == saved["attempt_id"]])
report = None
if phase == "finalizing":
report = _one([item["artifact"] for step in saved["steps"]
for item in step["artifacts"] if item["role"] == self._report_role
and item["status"] == "available"
and item["attempt_id"] == saved["attempt_id"]
and item["step_id"] == step["step_id"]])
return self._project(
phase, saved, candidate_bytes=_read(candidate, artifacts),
report_bytes=_read(report, artifacts),
)
4 changes: 3 additions & 1 deletion tests/check_installed.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import inferrsi
import inferrsi.harness
from inferrsi.apps.api import CandidateProvider, EvaluationPort
from inferrsi.apps.api import CandidateProvider, EvaluationPort, EvidenceCapture
from inferrsi.apps.cli import main
from inferrsi.benchmark.registry import builtin_tasks, describe_task, task_resources
from inferrsi.benchmark.skill_eval_engineering import REQUEST
Expand All @@ -27,6 +27,8 @@
from inferrsi.resources import repository_packs
from inferrsi.resources.repository_packs import REPO_TYPES, repository_pack

assert callable(EvidenceCapture)

# The new Harness binding port ships independently of Apps and Benchmark.
subprocess.run([
sys.executable, "-I", "-B", "-c",
Expand Down
168 changes: 168 additions & 0 deletions tests/test_apps_capture_adapter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
"""Role selection and byte authority in the thin Apps capture adapter."""

from copy import deepcopy
from hashlib import sha256
from io import BytesIO

import pytest

from inferrsi.apps import api
from inferrsi.harness.errors import PublicError


@pytest.fixture
def supplied():
data = {"candidate": b"actual candidate", "report": b"actual report"}
refs = {role: {"location": "retained/" + role, "sha256": sha256(raw).hexdigest(),
"size_bytes": len(raw), "media_type": "application/octet-stream"}
for role, raw in data.items()}
facts = {"attempt_id": "runtime-attempt", "candidate": {"input_role": "candidate"},
"bound_inputs": [{"role": "candidate", "attempt_id": "runtime-attempt",
"retained_ref": refs["candidate"]}],
"steps": [{"step_id": "main", "artifacts": [{
"role": "report", "attempt_id": "runtime-attempt", "step_id": "main",
"status": "available", "artifact": refs["report"],
}]}]}
calls, streams = [], []

class Reader:
def open(self, ref):
calls.append(deepcopy(ref))
source = BytesIO(data[ref["location"].split("/")[-1]])
streams.append(source)
return source

return data, refs, facts, Reader(), calls, streams


@pytest.mark.parametrize("phase", ["preparing", "finalizing"])
def test_roles_select_only_current_attempt_bytes_and_preserve_projector_result(supplied, phase):
data, refs, facts, reader, calls, streams = supplied
expected = object()

def project(actual_phase, actual_facts, *, candidate_bytes, report_bytes):
assert actual_phase == phase and actual_facts == facts
assert candidate_bytes == data["candidate"]
assert report_bytes == (data["report"] if phase == "finalizing" else None)
actual_facts["candidate"]["input_role"] = "mutated"
return expected

capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report")
assert capture.capture(phase, facts, artifacts=reader) is expected
assert calls == [refs["candidate"], *([refs["report"]] if phase == "finalizing" else [])]
assert facts["candidate"]["input_role"] == "candidate"
assert all(stream.closed for stream in streams)


@pytest.mark.parametrize("change", ["missing", "foreign-attempt", "foreign-step"])
def test_absent_or_foreign_report_is_not_used_as_current_evidence(supplied, change):
_, _, facts, reader, _, _ = supplied
if change == "missing":
facts["steps"][0]["artifacts"].clear()
else:
key = "attempt_id" if change == "foreign-attempt" else "step_id"
facts["steps"][0]["artifacts"][0][key] = "foreign"

def project(phase, actual_facts, *, candidate_bytes, report_bytes):
assert report_bytes is None and candidate_bytes is not None
return "unchanged result"

capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report")
assert capture.capture("finalizing", facts, artifacts=reader) == "unchanged result"


@pytest.mark.parametrize("kind", ["candidate", "report"])
def test_ambiguous_sources_are_not_resolved_by_first_or_last(supplied, kind):
_, _, facts, reader, _, _ = supplied
values = facts["bound_inputs"] if kind == "candidate" else facts["steps"][0]["artifacts"]
values.append(deepcopy(values[0]))
capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("ambiguous capture reached B"),
candidate_role="candidate", report_role="report")
with pytest.raises(PublicError) as caught:
capture.capture("finalizing", facts, artifacts=reader)
assert caught.value.code == "IdentityMismatch"


def test_wrong_configured_candidate_role_refuses_before_any_byte_read(supplied):
_, _, facts, reader, calls, _ = supplied
capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("foreign role reached B"),
candidate_role="other", report_role="report")
with pytest.raises(PublicError) as caught:
capture.capture("preparing", facts, artifacts=reader)
assert caught.value.code == "IdentityMismatch" and calls == []


def test_reader_cannot_mutate_authoritative_ref_metadata(supplied):
data, _, facts, _, _, _ = supplied
before = deepcopy(facts)

class MutatingReader:
def open(self, ref):
ref["sha256"] = "b" * 64
return data["candidate"]

capture = api.EvidenceCapture(lambda *a, **k: "unchanged", candidate_role="candidate",
report_role="report")
assert capture.capture("preparing", facts, artifacts=MutatingReader()) == "unchanged"
assert facts == before


def test_corrupt_capability_bytes_raise_integrity_error_and_close_stream(supplied):
data, _, facts, reader, _, streams = supplied
data["candidate"] = b"tampered"
capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("unverified bytes reached B"),
candidate_role="candidate", report_role="report")
with pytest.raises(PublicError) as caught:
capture.capture("preparing", facts, artifacts=reader)
assert caught.value.code == "ArtifactIntegrityError" and streams[0].closed


def test_host_programming_errors_are_not_relabelled_as_evidence_gaps(supplied):
_, _, facts, _, _, _ = supplied

class Broken:
def open(self, ref):
raise TypeError("synthetic host bug")

capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("broken host reached B"),
candidate_role="candidate", report_role="report")
with pytest.raises(TypeError, match="synthetic host bug"):
capture.capture("preparing", facts, artifacts=Broken())


def test_short_binary_reads_are_collected_and_the_stream_is_closed(supplied):
data, _, facts, _, _, _ = supplied

class Short(BytesIO):
def read(self, size):
return super().read(min(size, 2))

stream = Short(data["candidate"])

class Reader:
def open(self, ref):
return stream

def project(phase, facts, *, candidate_bytes, report_bytes):
assert candidate_bytes == data["candidate"] and report_bytes is None
return "unchanged"

capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="report")
assert capture.capture("preparing", facts, artifacts=Reader()) == "unchanged"
assert stream.closed


@pytest.mark.parametrize("code", ["ArtifactUnavailable", "ArtifactIntegrityError",
"UnsafeArtifactPath"])
def test_capability_refusals_keep_their_original_code_for_harness(supplied, code):
_, _, facts, _, _, _ = supplied

class Refused:
def open(self, ref):
raise PublicError(code, stage="evidence_capture", message="synthetic refusal")

capture = api.EvidenceCapture(lambda *a, **k: pytest.fail("refused source reached B"),
candidate_role="candidate", report_role="report")
with pytest.raises(PublicError) as caught:
capture.capture("preparing", facts, artifacts=Refused())
assert caught.value.code == code
Loading
Loading