From bfa4bc23a4cd5f5ec031eb77b6b8989d10273987 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 19:29:15 +0800 Subject: [PATCH 1/8] feat(harness): validate prepared bindings before registration Adapt the authorized pool draft and freeze the null-timeout binding vector. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/apps-module-migrations.json | 1 + src/inferrsi/harness/execution_service.py | 84 +++++- src/inferrsi/harness/prepared_binding.py | 269 ++++++++++++++++++ .../harness-prepared-binding/README.md | 14 + .../d023-null-timeout.json | 1 + tests/test_harness_execute.py | 4 +- tests/test_harness_prepared_binding.py | 116 ++++++++ 7 files changed, 480 insertions(+), 9 deletions(-) create mode 100644 src/inferrsi/harness/prepared_binding.py create mode 100644 tests/fixtures/harness-prepared-binding/README.md create mode 100644 tests/fixtures/harness-prepared-binding/d023-null-timeout.json create mode 100644 tests/test_harness_prepared_binding.py diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index ebaf1b65c..a74372bf8 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -127,6 +127,7 @@ ], "consumers": [ "docs/apps-output.md", + "tests/test_harness_prepared_binding.py", "docs/architecture.md", "docs/module-layout.md", "docs/operations.md", diff --git a/src/inferrsi/harness/execution_service.py b/src/inferrsi/harness/execution_service.py index edc40a3fa..66995746e 100644 --- a/src/inferrsi/harness/execution_service.py +++ b/src/inferrsi/harness/execution_service.py @@ -12,11 +12,12 @@ from pathlib import Path from inferrsi.contracts import json_v1 -from inferrsi.harness import input_binding, local_process, run_store +from inferrsi.harness import input_binding, local_process, prepared_binding, run_store from inferrsi.harness.errors import PublicError -# H02 single local-CPU execute in ordinary command mode (V05 null set). Evaluation mode, -# entry-code digests (T37) and generation chaining are later slices and are refused here. +# H02 single local-CPU execute in ordinary command mode (V05 null set). Evaluation-mode +# specs are checked before registration (V05/V28/V32 and their fixed preparation) and then +# refused: the T37 entry-code check and generation chaining are later slices. ATTEMPT_FIELDS = ( "request_id", "run_id", "attempt_id", "owner", "target_request", "deadline_utc", "resources", "budget", "retry_of", "retry_reason", @@ -42,6 +43,10 @@ ) _OUTPUT_FIELDS = frozenset({"role", "path", "required", "media_type", "format_schema"}) _REF_FIELDS = frozenset({"location", "sha256", "size_bytes", "media_type"}) +_PREPARED_REF_FIELDS = frozenset({"prepared_id", "schema_version", "artifact"}) +_CANDIDATE_FIELDS = frozenset( + {"candidate_id", "kind", "input_role", "parent_id", "parent_digest", "context_refs"} +) _DIMENSION_FIELDS = frozenset({"dimension", "unit", "limit", "reserve_estimate"}) _DEFINITION_FIELDS = frozenset({"name", "revision"}) _DECIMAL = re.compile(r"(?:0|-?[1-9][0-9]*|-?(?:0|[1-9][0-9]*)\.[0-9]*[1-9])", re.ASCII) @@ -222,6 +227,9 @@ def _step_errors(steps: object, ordinary: bool) -> list[str]: elif ordinary and digest is not None: # V32-N1: ordinary command mode carries no expected entry digest. errors.append(f"{where}.expected_code_sha256: must be null in ordinary command mode") + elif not ordinary and digest is None: + # V32-N2: null never opts an evaluation step out of the frozen code digest. + errors.append(f"{where}.expected_code_sha256: must be set in evaluation mode (V32)") if not (isinstance(step["args"], list) and all(isinstance(a, str) for a in step["args"])): errors.append(f"{where}.args: not a list of Text") if step["cwd"] != "." and not _is_path(step["cwd"]): @@ -265,6 +273,53 @@ def _output_errors(outputs: object, where: str) -> list[str]: return errors +def _evaluation_errors(spec: Mapping) -> list[str]: + """Static shape of the evaluation fields; the fixed preparation is read separately.""" + errors: list[str] = [] + ref = spec["prepared_ref"] + if not run_store._has_fields(ref, _PREPARED_REF_FIELDS): + errors.append("prepared_ref: fields must be exactly the PreparedRef fields") + else: + if not prepared_binding._is_semantic_id(ref["prepared_id"]): + errors.append("prepared_ref.prepared_id: not a SemanticId") + if type(ref["schema_version"]) is not int or ref["schema_version"] != 1: + errors.append("prepared_ref.schema_version: must be the integer 1") + if not _is_artifact_ref(ref["artifact"]): + errors.append("prepared_ref.artifact: not an ArtifactRef") + elif ref["artifact"]["media_type"] != prepared_binding.PREPARED_MEDIA_TYPE: + errors.append("prepared_ref.artifact.media_type: must be application/json") + if not prepared_binding._is_semantic_id(spec["protocol_digest"]): + errors.append("protocol_digest: not a SemanticId") + for name in ("operation", "validation_scope"): + if not run_store._is_id(spec[name]): + errors.append(f"{name}: not an Id") + candidate = spec["candidate"] + if not run_store._has_fields(candidate, _CANDIDATE_FIELDS): + errors.append("candidate: fields must be exactly the CandidateSubmission fields") + return errors + for name in ("candidate_id", "kind", "input_role"): + if not run_store._is_id(candidate[name]): + errors.append(f"candidate.{name}: not an Id") + # V02: parent id and digest are both null or both set. + if (candidate["parent_id"] is None) != (candidate["parent_digest"] is None): + errors.append("candidate.parent_id: parent_id and parent_digest must be paired") + elif candidate["parent_id"] is not None and not ( + run_store._is_id(candidate["parent_id"]) and prepared_binding._is_sha256( + candidate["parent_digest"] + ) + ): + errors.append("candidate.parent_id: not an Id with a Sha256 parent_digest") + refs = candidate["context_refs"] + if not isinstance(refs, list) or not all(_is_artifact_ref(item) for item in refs): + errors.append("candidate.context_refs: not a list of ArtifactRef") + # V03: the candidate role names exactly one InputSpec. + inputs = spec["inputs"] if isinstance(spec["inputs"], list) else [] + roles = [item["role"] for item in inputs if isinstance(item, Mapping) and "role" in item] + if roles.count(candidate["input_role"]) != 1: + errors.append("candidate.input_role: must name exactly one input role (V03)") + return errors + + def _request_errors(spec: Mapping) -> list[str]: """Shape and resource/budget field errors beyond the AttemptContext rules.""" errors: list[str] = [] @@ -304,8 +359,12 @@ def _request_errors(spec: Mapping) -> list[str]: return errors -def validate(spec: object, target: LocalCpuTarget) -> None: - """Refuse a request before registration (D021 priority: version, fields, support).""" +def validate(spec: object, target: LocalCpuTarget, *, artifact_reader=None) -> None: + """Refuse a request before registration (D021 priority: version, fields, support). + + An evaluation-mode spec is also checked against its fixed preparation, read through + ``artifact_reader`` (V05, V28, V32). Nothing here registers an attempt or writes a record. + """ if isinstance(spec, Mapping): version = spec.get("schema_version") if type(version) is int and version != 1: @@ -323,8 +382,11 @@ def validate(spec: object, target: LocalCpuTarget) -> None: errors += run_store._attempt_errors({name: spec[name] for name in ATTEMPT_FIELDS}) errors += _request_errors(spec) evaluation = [name for name in _EVALUATION_FIELDS if spec[name] is not None] - if evaluation and len(evaluation) != len(_EVALUATION_FIELDS): + complete = len(evaluation) == len(_EVALUATION_FIELDS) + if evaluation and not complete: errors.append("prepared_ref: evaluation fields must be all null or all set (V05)") + elif complete: + errors += _evaluation_errors(spec) errors += _input_errors(spec["inputs"]) errors += _step_errors(spec["steps"], ordinary=not evaluation) try: @@ -336,9 +398,15 @@ def validate(spec: object, target: LocalCpuTarget) -> None: "InvalidExecutionSpec", stage="validate", message="illegal ExecutionSpec", field_errors=errors, ) + if evaluation: + # Malformed or inconsistent bindings are refused here, before the RunStore sees the + # request. The reader is the read-only port; H does not write or register anything. + prepared_binding.verify(spec, artifact_reader) unsupported = [] if evaluation: - unsupported.append("evaluation mode is not implemented in this slice") + # The T37 pre-start entry-code check runs after registration and is a later slice. + # Until it exists, no evaluation-mode candidate may start. + unsupported.append("evaluation-mode entry verification is not implemented in this slice") if spec["generation_record_ref"] is not None: unsupported.append("generation chaining is not implemented in this slice") request = spec["target_request"] @@ -920,7 +988,7 @@ def execute( artifact_reader=None, ) -> dict: """H ``execute``: register, admit, bind, run and commit one ordinary command request.""" - validate(spec, target) + validate(spec, target, artifact_reader=artifact_reader) digest = run_store.request_digest(spec) attempt = {name: copy.deepcopy(spec[name]) for name in ATTEMPT_FIELDS} # Re-entry is decided by the store before any binding or start (V14 / D021 section 3). diff --git a/src/inferrsi/harness/prepared_binding.py b/src/inferrsi/harness/prepared_binding.py new file mode 100644 index 000000000..a3ab05ad5 --- /dev/null +++ b/src/inferrsi/harness/prepared_binding.py @@ -0,0 +1,269 @@ +"""H's read-only binding projection of a fixed PreparedEvaluation (D021 section 1, V28, V32). + +An evaluation-mode ExecutionSpec names its preparation by PreparedRef. Before any +registration, H reads the fixed bytes through the read-only artifact port, verifies them +against the Ref and compares the fields it needs with the spec: ids, digests, operation and +scope, the candidate declaration and the frozen worker code digest. The projection is defined +here. It imports nothing from Benchmark or Apps, and it never interprets task rules, formats +or scores. + +Nothing in this module registers an attempt, writes a record or starts a process. Every +refusal is raised before the RunStore sees the request. Verifying the actual entry bytes +against the frozen digest (T37) happens after registration and is not part of this module. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Mapping +from dataclasses import dataclass + +from inferrsi.contracts import json_v1 +from inferrsi.harness import run_store +from inferrsi.harness.errors import PublicError + +STAGE = "prepared_binding" +PREPARED_MEDIA_TYPE = "application/json" +# The candidate identity rules this H supports (V28). Any other is refused, never downgraded. +CAPTURED_SHA256 = {"name": "captured-sha256", "revision": "1"} +EXPECTED_SHA256 = {"name": "expected-sha256", "revision": "1"} + +# Exact v1 field sets of the objects the projection reads. An unknown field is refused and +# never dropped (V01). +_PREPARED_FIELDS = frozenset( + { + "schema_version", "prepared_id", "task", "grader", "operation", "validation_scope", + "protocol", "protocol_digest", "candidate_contract", "expected_candidate", + "required_evidence", "recipe", "resources", "created_at", "display_notes", + } +) +_CONTRACT_FIELDS = frozenset( + { + "kind", "input_role", "format_schema", "entrypoint", "required_paths", + "allowed_content", "identity_rule", "modification_receipt_required", + } +) +_RECIPE_FIELDS = frozenset( + { + "recipe_revision", "runtime_role", "entrypoint_kind", "entrypoint", + "entrypoint_revision", "code_sha256", "args", "input_roles", "outputs", + "required_capabilities", + } +) +_EXPECTED_FIELDS = frozenset({"candidate_id", "sha256", "format_schema"}) +_DEFINITION_FIELDS = frozenset({"name", "revision"}) +_SEMANTIC_ID = re.compile(r"sha256:[0-9a-f]{64}", re.ASCII) +_SHA256 = re.compile(r"[0-9a-f]{64}", re.ASCII) +_READ_CHUNK = 1024 * 1024 + + +@dataclass(frozen=True) +class FixedPreparation: + """The identity fields H reads from a fixed preparation; nothing else is retained.""" + + prepared_id: str + operation: str + validation_scope: str + protocol_digest: str + candidate_kind: str + candidate_input_role: str + candidate_format: dict + identity_rule: dict + expected_candidate: dict | None + code_sha256: str + + +def _refuse(code: str, message: str, field_errors: list[str] | None = None) -> PublicError: + return PublicError(code, stage=STAGE, message=message, field_errors=field_errors or []) + + +def _is_definition(value: object) -> bool: + return run_store._has_fields(value, _DEFINITION_FIELDS) and all( + run_store._is_id(value[name]) for name in _DEFINITION_FIELDS + ) + + +def _is_sha256(value: object) -> bool: + return isinstance(value, str) and _SHA256.fullmatch(value) is not None + + +def _is_semantic_id(value: object) -> bool: + return isinstance(value, str) and _SEMANTIC_ID.fullmatch(value) is not None + + +def read(ref: Mapping, reader) -> bytes: + """The fixed bytes of ``ref`` from the read-only port, verified against the Ref. + + The port's own verification is not trusted: size and hash are checked here again + (V06). At most ``size_bytes + 1`` bytes are read, so a wrong Ref cannot make this + allocate more than the stream actually holds. + """ + if reader is None: + raise _refuse( + "ArtifactUnavailable", "no read-only artifact reader is configured for the " + "fixed preparation", + ) + size = ref["size_bytes"] + digest = hashlib.sha256() + parts: list[bytes] = [] + total = 0 + try: + stream = reader.open(ref) + with stream: + while total <= size: + chunk = stream.read(min(_READ_CHUNK, size + 1 - total)) + if not chunk: + break + digest.update(chunk) + parts.append(chunk) + total += len(chunk) + except PublicError: + raise + except OSError as exc: + raise _refuse( + "ArtifactUnavailable", + f"the fixed preparation could not be read: {run_store._describe(exc)}", + ) from exc + if total != size or digest.hexdigest() != ref["sha256"]: + raise _refuse( + "ArtifactIntegrityError", "the fixed preparation bytes differ from their ArtifactRef" + ) + return b"".join(parts) + + +def project(data: bytes) -> FixedPreparation: + """Parse the identity fields of canonical PreparedEvaluation bytes.""" + try: + prepared = json_v1.decode(data) + except json_v1.CanonicalError as exc: + raise _refuse( + "ArtifactIntegrityError", + f"the fixed preparation is not canonical inferrsi-json-v1 ({exc.stage})", + ) from exc + version = prepared.get("schema_version") if isinstance(prepared, dict) else None + if type(version) is int and version != 1: + raise _refuse("UnsupportedSchema", "only PreparedEvaluation v1 is supported") + errors: list[str] = [] + if not run_store._has_fields(prepared, _PREPARED_FIELDS): + raise _refuse( + "InvalidExecutionSpec", "illegal fixed preparation", + ["prepared: fields must be exactly the PreparedEvaluation fields"], + ) + if type(version) is not int: + errors.append("prepared.schema_version: must be the integer 1") + if not _is_semantic_id(prepared["prepared_id"]): + errors.append("prepared.prepared_id: not a SemanticId") + if not _is_semantic_id(prepared["protocol_digest"]): + errors.append("prepared.protocol_digest: not a SemanticId") + for name in ("operation", "validation_scope"): + if not run_store._is_id(prepared[name]): + errors.append(f"prepared.{name}: not an Id") + contract, recipe = prepared["candidate_contract"], prepared["recipe"] + expected = prepared["expected_candidate"] + if not run_store._has_fields(contract, _CONTRACT_FIELDS): + errors.append("prepared.candidate_contract: not the CandidateContract fields") + else: + for name in ("kind", "input_role"): + if not run_store._is_id(contract[name]): + errors.append(f"prepared.candidate_contract.{name}: not an Id") + for name in ("format_schema", "identity_rule"): + if not _is_definition(contract[name]): + errors.append(f"prepared.candidate_contract.{name}: not a DefinitionRef") + if not run_store._has_fields(recipe, _RECIPE_FIELDS): + errors.append("prepared.recipe: not the OperationRecipe fields") + elif not _is_sha256(recipe["code_sha256"]): + errors.append("prepared.recipe.code_sha256: not a Sha256") + if expected is not None: + if not run_store._has_fields(expected, _EXPECTED_FIELDS): + errors.append( + "prepared.expected_candidate: not null or the CandidateExpectation fields" + ) + else: + if not run_store._is_id(expected["candidate_id"]): + errors.append("prepared.expected_candidate.candidate_id: not an Id") + if not _is_sha256(expected["sha256"]): + errors.append("prepared.expected_candidate.sha256: not a Sha256") + if not _is_definition(expected["format_schema"]): + errors.append("prepared.expected_candidate.format_schema: not a DefinitionRef") + if errors: + raise _refuse("InvalidExecutionSpec", "illegal fixed preparation", errors) + return FixedPreparation( + prepared_id=prepared["prepared_id"], + operation=prepared["operation"], + validation_scope=prepared["validation_scope"], + protocol_digest=prepared["protocol_digest"], + candidate_kind=contract["kind"], + candidate_input_role=contract["input_role"], + candidate_format=contract["format_schema"], + identity_rule=contract["identity_rule"], + expected_candidate=expected, + code_sha256=recipe["code_sha256"], + ) + + +def _check_identity(spec: Mapping, fixed: FixedPreparation) -> None: + """V17: top-level identity fields must agree before any declaration is compared.""" + pairs = ( + ("prepared_ref.prepared_id", spec["prepared_ref"]["prepared_id"], fixed.prepared_id), + ("protocol_digest", spec["protocol_digest"], fixed.protocol_digest), + ("operation", spec["operation"], fixed.operation), + ("validation_scope", spec["validation_scope"], fixed.validation_scope), + ) + differing = [f"{name}: differs from the fixed preparation" for name, a, b in pairs if a != b] + if differing: + raise _refuse( + "IdentityMismatch", "the execution spec names a different preparation", differing + ) + + +def _check_declarations(spec: Mapping, fixed: FixedPreparation) -> None: + """V28 and V32: declaration conflicts are refused before registration, never recorded.""" + errors = [ + f"steps[{index}].expected_code_sha256: must equal the frozen recipe code_sha256 (V32)" + for index, step in enumerate(spec["steps"]) + if step["expected_code_sha256"] != fixed.code_sha256 + ] + candidate = spec["candidate"] + if candidate["input_role"] != fixed.candidate_input_role: + errors.append("candidate.input_role: differs from the frozen candidate contract") + if candidate["kind"] != fixed.candidate_kind: + errors.append("candidate.kind: differs from the frozen candidate contract") + declared = next( + (item["expected_sha256"] for item in spec["inputs"] + if item["role"] == candidate["input_role"]), + None, + ) + if fixed.identity_rule == EXPECTED_SHA256 and declared is None: + errors.append( + "inputs: the candidate input needs expected_sha256 under expected-sha256@1 (V28)" + ) + frozen = fixed.expected_candidate + if frozen is not None: + if candidate["candidate_id"] != frozen["candidate_id"]: + errors.append("candidate.candidate_id: differs from the frozen expected candidate") + if declared != frozen["sha256"]: + errors.append("inputs: the candidate expected_sha256 differs from the frozen one") + if fixed.candidate_format != frozen["format_schema"]: + errors.append("prepared: expected candidate format differs from the candidate contract") + if errors: + raise _refuse( + "InvalidExecutionSpec", "the execution spec conflicts with its fixed preparation", + errors, + ) + + +def verify(spec: Mapping, reader) -> FixedPreparation: + """Check an evaluation-mode spec against its fixed preparation before registration. + + ``spec`` has passed the static ExecutionSpec checks, so its evaluation fields are all + set and well formed. Raises a PublicError for any malformed or inconsistent binding. + """ + fixed = project(read(spec["prepared_ref"]["artifact"], reader)) + if fixed.identity_rule not in (CAPTURED_SHA256, EXPECTED_SHA256): + raise _refuse( + "UnsupportedRuleVersion", "the frozen candidate identity rule is unsupported" + ) + _check_identity(spec, fixed) + _check_declarations(spec, fixed) + return fixed diff --git a/tests/fixtures/harness-prepared-binding/README.md b/tests/fixtures/harness-prepared-binding/README.md new file mode 100644 index 000000000..e235960c0 --- /dev/null +++ b/tests/fixtures/harness-prepared-binding/README.md @@ -0,0 +1,14 @@ +# D023 null-timeout binding vector + +`d023-null-timeout.json` is derived from the unchanged canonical +`docs/contracts/fixtures/t108-v4/execution-spec.json`: add the frozen preparation's +`recipe.code_sha256` as `steps[0].expected_code_sha256`, and set +`steps[0].timeout_seconds` to null. The accepted #429 PR-1 vector is 2069 bytes, +with request digest +`sha256:62ce11e6d458b5c581245d0988189e1bd834f2ed1e1880d15242faf70c5b6fe9`. + +This is immutable contract-conformance input, not observed execution evidence or +an available worker implementation. Tests compare the canonical Apps binder, +Harness digest and read-only prepared projection. No process is started. T37 +entry verification and production T40 replay remain separate PR-2/PR-3 work. +The original documentary fixtures are not rewritten. diff --git a/tests/fixtures/harness-prepared-binding/d023-null-timeout.json b/tests/fixtures/harness-prepared-binding/d023-null-timeout.json new file mode 100644 index 000000000..da758ae6b --- /dev/null +++ b/tests/fixtures/harness-prepared-binding/d023-null-timeout.json @@ -0,0 +1 @@ +{"attempt_id":"att-1","budget":{"declared_by":"fixture","dimensions":[],"policy_ref":null,"required_enforcement":[],"scope_id":"campaign-1"},"candidate":{"candidate_id":"cand-1","context_refs":[],"input_role":"candidate","kind":"text","parent_digest":null,"parent_id":null},"deadline_utc":"2099-01-01T00:00:00Z","generation_record_ref":null,"inputs":[{"expected_sha256":"e39494a6eed78161414f7afdef602f86ff19de6efa9ca9053407d548981fe01f","materialized_path":"inputs/candidate.txt","role":"candidate","snapshot_format":"file-v1","source":"fixtures/candidate.txt","source_namespace":"artifact","source_ref":{"location":"fixtures/candidate.txt","media_type":"text/plain","sha256":"e39494a6eed78161414f7afdef602f86ff19de6efa9ca9053407d548981fe01f","size_bytes":18}}],"operation":"evaluate","output_root":"attempts/att-1","owner":"fixture","prepared_ref":{"artifact":{"location":"fixtures/prepared.json","media_type":"application/json","sha256":"8450cf590172a785282fed31480eb82bb192f4e9a72c73dd70d0b5ea9a3d30ff","size_bytes":2690},"prepared_id":"sha256:7f481e7e61c98343015118d7dcd98806abe863398b2c16dacac125d2d00ed486","schema_version":1},"protocol_digest":"sha256:497290081c4f3d3cab0c50c42a0be1c467e1a840dcf680eaace602311c46a80c","request_id":"att-1","required_capabilities":["local-process-tree"],"resources":{"cpu_count":1,"memory_bytes":null,"required_capabilities":["local-process-tree"]},"retry_of":null,"retry_reason":null,"run_id":"run-1","schema_version":1,"steps":[{"args":["inputs/candidate.txt","outputs/grader.json"],"cwd":".","entrypoint":"fixture_worker","entrypoint_kind":"python_module","entrypoint_revision":"1","expected_code_sha256":"ba351e8f96c162ed9c32309f24b1d2d75d11a7369168571ebbb0a79d13315332","outputs":[{"format_schema":{"name":"fixture-grader","revision":"1"},"media_type":"application/json","path":"outputs/grader.json","required":true,"role":"grader"}],"runtime_role":"local-python","step_id":"main","timeout_seconds":null}],"target_request":{"config_revision":"1","executor_kind":"local_cpu","target_id":"fixture-cpu"},"validation_scope":"p1"} \ No newline at end of file diff --git a/tests/test_harness_execute.py b/tests/test_harness_execute.py index efd054515..0778e0d05 100644 --- a/tests/test_harness_execute.py +++ b/tests/test_harness_execute.py @@ -37,6 +37,7 @@ NEW_H_CLOSURE = { "inferrsi.harness.api", "inferrsi.harness.execution_service", "inferrsi.harness.input_binding", "inferrsi.harness.local_process", + "inferrsi.harness.prepared_binding", "inferrsi.harness.atomic_json", "inferrsi.harness.run_store", "inferrsi.harness.errors", "inferrsi.harness.execution_spec", "inferrsi.contracts.json_v1", @@ -436,7 +437,8 @@ def test_spec_time_refusals_register_nothing(h: Harness) -> None: candidate={"candidate_id": "cand", "context_refs": [], "input_role": "candidate", "kind": "text", "parent_digest": None, "parent_id": None}, ) - assert error_of(h.execute, evaluation).code == "UnsupportedOperation" + # The malformed binding is rejected before support/entry checks (PR-1 / V32-N2). + assert error_of(h.execute, evaluation).code == "InvalidExecutionSpec" assert error_of(h.execute, dict(spec, schema_version=2)).code == "UnsupportedSchema" float_timeout = dict(spec, steps=[dict(spec["steps"][0], timeout_seconds=1.5)]) assert error_of(h.execute, float_timeout).code == "InvalidExecutionSpec" diff --git a/tests/test_harness_prepared_binding.py b/tests/test_harness_prepared_binding.py new file mode 100644 index 000000000..4601ebac3 --- /dev/null +++ b/tests/test_harness_prepared_binding.py @@ -0,0 +1,116 @@ +"""PR-1: bind frozen preparation before registration; never perform T37 entry checks.""" + +import sqlite3 +from copy import deepcopy +from hashlib import sha256 +from io import BytesIO +from pathlib import Path + +import pytest + +from inferrsi.apps.handoff.bridge import bind_execution, verify_protocol +from inferrsi.contracts.json_v1 import decode, encode +from inferrsi.harness import execution_service, prepared_binding, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from inferrsi.harness.errors import PublicError + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = ROOT / "docs/contracts/fixtures/t108-v4" +VECTOR = ROOT / "tests/fixtures/harness-prepared-binding/d023-null-timeout.json" +DIGEST = "sha256:62ce11e6d458b5c581245d0988189e1bd834f2ed1e1880d15242faf70c5b6fe9" + + +def test_d023_null_timeout_bytes_match_apps_and_harness(): + data = VECTOR.read_bytes() + spec = decode(data) + prepared = decode((GOLDEN / "prepared.json").read_bytes()) + verify_protocol(prepared, (GOLDEN / "protocol.json").read_bytes()) + request = { + "schema_version": 1, "prepared_ref": spec["prepared_ref"], + "attempt": {key: spec[key] for key in execution_service.ATTEMPT_FIELDS}, + "proposal": {key: spec[key] for key in ("candidate", "inputs", "generation_record_ref")}, + "output_root": spec["output_root"], + } + assert len(data) == 2069 + assert "sha256:" + sha256(data).hexdigest() == DIGEST + assert run_store.request_digest(spec) == DIGEST + assert encode(bind_execution(request, prepared)) == data + assert spec["steps"][0]["timeout_seconds"] is None + + +@pytest.fixture +def binding(tmp_path, monkeypatch): + spec = decode(VECTOR.read_bytes()) + prepared = (GOLDEN / "prepared.json").read_bytes() + + class Reader: + def __init__(self): + self.opens = [] + self.data = prepared + + def open(self, ref): + assert ref == spec["prepared_ref"]["artifact"] + self.opens.append(deepcopy(ref)) + return BytesIO(self.data) + + reader = Reader() + store = run_store.RunStore(tmp_path / "store", "binding-tests") + target = LocalCpuTarget("fixture-cpu", "1", work_root=tmp_path / "work") + harness = Harness(store, target, artifact_reader=reader) + + def forbidden(*args, **kwargs): + pytest.fail("a PR-1 refusal reached registration or process execution") + + monkeypatch.setattr(store, "claim", forbidden) + monkeypatch.setattr(execution_service.local_process, "execute_command", forbidden) + return spec, reader, store, harness + + +def assert_empty(store): + with sqlite3.connect(store.root / run_store.INDEX_NAME) as connection: + assert connection.execute("SELECT count(*) FROM attempts").fetchone()[0] == 0 + assert connection.execute("SELECT count(*) FROM records").fetchone()[0] == 0 + + +def test_projection_reads_the_fixed_preparation_without_resolving_an_entry(binding): + spec, reader, store, harness = binding + fixed = prepared_binding.verify(spec, reader) + assert fixed.prepared_id == spec["prepared_ref"]["prepared_id"] + assert fixed.code_sha256 == spec["steps"][0]["expected_code_sha256"] + assert len(reader.opens) == 1 + # T37 is a later delivery: valid evaluation bindings still cannot start. + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "UnsupportedOperation" + assert_empty(store) + + +@pytest.mark.parametrize("failure", ["null-code", "wrong-code", "protocol", "candidate", "input"]) +def test_malformed_or_inconsistent_binding_registers_no_attempt(binding, failure): + spec, reader, store, harness = binding + if failure == "null-code": + spec["steps"][0]["expected_code_sha256"] = None + elif failure == "wrong-code": + spec["steps"][0]["expected_code_sha256"] = "0" * 64 + elif failure == "protocol": + spec["protocol_digest"] = "sha256:" + "0" * 64 + elif failure == "candidate": + spec["candidate"]["candidate_id"] = "different-candidate" + else: + spec["inputs"][0]["expected_sha256"] = "0" * 64 + spec["inputs"][0]["source_ref"]["sha256"] = "0" * 64 + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == ("IdentityMismatch" if failure == "protocol" + else "InvalidExecutionSpec") + assert_empty(store) + assert len(reader.opens) == (0 if failure == "null-code" else 1) + + +def test_corrupt_preparation_bytes_cannot_reach_registration(binding): + spec, reader, store, harness = binding + reader.data = b"corrupt retained preparation" + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "ArtifactIntegrityError" + assert_empty(store) From b8d8685c74fe4c4a116e06b178e9bc32060605e3 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 19:45:39 +0800 Subject: [PATCH 2/8] feat(harness): bind frozen worker declarations before registration Verify candidate and worker declarations against fixed preparation bytes. Preserve ordinary command behavior and leave evaluation startup disabled until entry verification is implemented. Agent-assisted-by: Codex (author) Agent-assisted-by: Unknown PR Pool agent (draft contributor) Signed-off-by: zengchuang --- docs/contracts/harness-prepared-binding.md | 52 +++++ docs/operations.md | 1 + src/inferrsi/harness/prepared_binding.py | 112 +++++++++- tests/check_installed.py | 9 + tests/test_harness_prepared_binding.py | 240 +++++++++++++++++++++ 5 files changed, 405 insertions(+), 9 deletions(-) create mode 100644 docs/contracts/harness-prepared-binding.md diff --git a/docs/contracts/harness-prepared-binding.md b/docs/contracts/harness-prepared-binding.md new file mode 100644 index 000000000..15db9301e --- /dev/null +++ b/docs/contracts/harness-prepared-binding.md @@ -0,0 +1,52 @@ +# Harness preparation binding before registration + +The Harness evaluation path checks declarations before registering an attempt. +`inferrsi.harness.prepared_binding.verify(spec, artifact_reader)` returns a +`FixedPreparation` projection or raises a Harness `PublicError`. The spec must +first pass `execution_service.validate`'s static shape checks; `Harness.execute` +performs both checks in that order. Ordinary command mode reads no preparation. + +The projection's `read(ref, reader)` port obtains a binary stream through +`reader.open(ArtifactRef)`, closes it, bounds reads by the declared size plus one +byte, and independently checks size and SHA-256. Missing bytes or a missing +reader produce `ArtifactUnavailable`; changed bytes or noncanonical JSON +produce `ArtifactIntegrityError`. Unknown host programming errors propagate. +`project(bytes)` is pure: it parses the canonical v1 envelope and validates the +fields used by Harness. It does not read a protocol, resolve an entry or start a +process. A schema version other than v1 is explicitly unsupported. + +Binding compares the semantic prepared ID, protocol digest, operation and scope +with the fixed preparation. Disagreement is `IdentityMismatch`. It also checks: + +- Candidate kind and role, a frozen expected candidate ID/hash/format when + present, and the supported `captured-sha256@1` or `expected-sha256@1` identity + rule. An unknown rule/revision is `UnsupportedRuleVersion`. The expected rule + requires a declared expected input hash even without a frozen candidate. +- A non-null expected code digest on every evaluation step, matching the frozen + recipe. This is an expected declaration, not an observed entry-code hash. +- One worker step preserving runtime role, entry kind/name/revision, all output + declarations, and arguments with whole-argument `{input:role}` / `{output:role}` + expansion. Inputs may use caller-chosen materialized paths; arguments must + match those paths. Paths are relative to the attempt directory (`cwd="."`), + and input/output paths must not overlap. Required frozen capabilities cannot + be dropped; additional declared capabilities remain subject to admission. + +Malformed or conflicting declarations are `InvalidExecutionSpec`. All of these +refusals precede `RunStore.claim`: neither attempt nor record is registered. +The public projection does not interpret candidate formats, grading rules, +evidence requirements or scores, and does not recompute Benchmark's semantic +prepared ID. The fixed ArtifactRef authenticates the preparation bytes; Apps +performs the separate archived-protocol binding through its canonical +`inferrsi.apps.handoff.bridge.verify_protocol` port. + +The D023 conformance vector is kept separately under +`tests/fixtures/harness-prepared-binding/`. It adds the frozen code declaration +and a null step timeout without rewriting historical fixtures. Its canonical +bytes must agree between the Apps binder and Harness `request_digest`. +The vector is documentary; its historical code pin is not an available worker. + +This slice keeps valid evaluation-mode execution explicitly unsupported. +Actual entry resolution and code-byte verification after registration (T37), +attempt deadlines/budget enforcement with a null step timeout, and evaluation +replay (T40) are later delivery slices. It does not establish production CPU +acceptance or completion of the governing Harness or Apps Issues. diff --git a/docs/operations.md b/docs/operations.md index 9115ca36b..4e4531453 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -52,6 +52,7 @@ Effect classes used below: | Rebuild scoped legacy evidence views | `python tools/rebuild_model_evidence.py --archive --output ` | CPU-only; creates a new output directory | Authenticates original bytes and pinned projection/code; no extraction, network, archived-code execution or campaign qualification | | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | +| Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | diff --git a/src/inferrsi/harness/prepared_binding.py b/src/inferrsi/harness/prepared_binding.py index a3ab05ad5..dbeb6709e 100644 --- a/src/inferrsi/harness/prepared_binding.py +++ b/src/inferrsi/harness/prepared_binding.py @@ -3,7 +3,7 @@ An evaluation-mode ExecutionSpec names its preparation by PreparedRef. Before any registration, H reads the fixed bytes through the read-only artifact port, verifies them against the Ref and compares the fields it needs with the spec: ids, digests, operation and -scope, the candidate declaration and the frozen worker code digest. The projection is defined +scope, the candidate declaration and the frozen worker declarations. The projection is defined here. It imports nothing from Benchmark or Apps, and it never interprets task rules, formats or scores. @@ -20,7 +20,7 @@ from dataclasses import dataclass from inferrsi.contracts import json_v1 -from inferrsi.harness import run_store +from inferrsi.harness import input_binding, run_store from inferrsi.harness.errors import PublicError STAGE = "prepared_binding" @@ -53,6 +53,9 @@ ) _EXPECTED_FIELDS = frozenset({"candidate_id", "sha256", "format_schema"}) _DEFINITION_FIELDS = frozenset({"name", "revision"}) +_OUTPUT_FIELDS = frozenset({"role", "path", "required", "media_type", "format_schema"}) +_ARTIFACT_FIELDS = frozenset({"location", "sha256", "size_bytes", "media_type"}) +_PLACEHOLDER = re.compile(r"\{(input|output):([^{}]+)\}") _SEMANTIC_ID = re.compile(r"sha256:[0-9a-f]{64}", re.ASCII) _SHA256 = re.compile(r"[0-9a-f]{64}", re.ASCII) _READ_CHUNK = 1024 * 1024 @@ -60,7 +63,7 @@ @dataclass(frozen=True) class FixedPreparation: - """The identity fields H reads from a fixed preparation; nothing else is retained.""" + """H's mechanical identity/binding projection; task rules and scores are not retained.""" prepared_id: str operation: str @@ -72,6 +75,7 @@ class FixedPreparation: identity_rule: dict expected_candidate: dict | None code_sha256: str + recipe: dict def _refuse(code: str, message: str, field_errors: list[str] | None = None) -> PublicError: @@ -92,6 +96,60 @@ def _is_semantic_id(value: object) -> bool: return isinstance(value, str) and _SEMANTIC_ID.fullmatch(value) is not None +def _is_ids(value: object) -> bool: + return (isinstance(value, list) and all(run_store._is_id(item) for item in value) + and len(set(value)) == len(value)) + + +def _is_path(value: object) -> bool: + return isinstance(value, str) and input_binding.RELATIVE_PATH.fullmatch(value) is not None + + +def _recipe_errors(recipe: object) -> list[str]: + """Validate only mechanical worker declarations, without interpreting formats or rules.""" + if not run_store._has_fields(recipe, _RECIPE_FIELDS): + return ["prepared.recipe: not the OperationRecipe fields"] + errors = [] + for name in ("recipe_revision", "runtime_role", "entrypoint_revision"): + if not run_store._is_id(recipe[name]): + errors.append(f"prepared.recipe.{name}: not an Id") + if recipe["entrypoint_kind"] not in ("python_module", "executable"): + errors.append("prepared.recipe.entrypoint_kind: not a supported entry kind") + if not isinstance(recipe["entrypoint"], str) or not recipe["entrypoint"]: + errors.append("prepared.recipe.entrypoint: not a non-empty Text") + if not _is_sha256(recipe["code_sha256"]): + errors.append("prepared.recipe.code_sha256: not a Sha256") + for name in ("input_roles", "required_capabilities"): + if not _is_ids(recipe[name]): + errors.append(f"prepared.recipe.{name}: not a list of unique Ids") + outputs = recipe["outputs"] + if not isinstance(outputs, list) or not all( + run_store._has_fields(item, _OUTPUT_FIELDS) + and run_store._is_id(item["role"]) and _is_path(item["path"]) + and type(item["required"]) is bool + and isinstance(item["media_type"], str) and bool(item["media_type"]) + and _is_definition(item["format_schema"]) for item in outputs + ): + errors.append("prepared.recipe.outputs: not a list of OutputSpec") + elif (len({item["role"] for item in outputs}) != len(outputs) + or len({item["path"] for item in outputs}) != len(outputs)): + errors.append("prepared.recipe.outputs: roles and paths must be unique") + args = recipe["args"] + if not isinstance(args, list) or not all(isinstance(arg, str) for arg in args): + errors.append("prepared.recipe.args: not a list of Text") + elif not errors: + roles = {"input": set(recipe["input_roles"]), + "output": {item["role"] for item in outputs}} + for arg in args: + match = _PLACEHOLDER.fullmatch(arg) + if match: + if match[2] not in roles[match[1]]: + errors.append("prepared.recipe.args: unknown placeholder role") + elif "{input:" in arg or "{output:" in arg: + errors.append("prepared.recipe.args: placeholders must occupy a whole argument") + return errors + + def read(ref: Mapping, reader) -> bytes: """The fixed bytes of ``ref`` from the read-only port, verified against the Ref. @@ -99,17 +157,23 @@ def read(ref: Mapping, reader) -> bytes: (V06). At most ``size_bytes + 1`` bytes are read, so a wrong Ref cannot make this allocate more than the stream actually holds. """ + if not (run_store._has_fields(ref, _ARTIFACT_FIELDS) and _is_path(ref["location"]) + and _is_sha256(ref["sha256"]) and type(ref["size_bytes"]) is int + and 0 <= ref["size_bytes"] <= json_v1.MAX_SAFE_INTEGER + and ref["media_type"] == PREPARED_MEDIA_TYPE): + raise _refuse("InvalidExecutionSpec", "illegal fixed preparation ArtifactRef") if reader is None: raise _refuse( "ArtifactUnavailable", "no read-only artifact reader is configured for the " "fixed preparation", ) size = ref["size_bytes"] + expected_sha256 = ref["sha256"] digest = hashlib.sha256() parts: list[bytes] = [] total = 0 try: - stream = reader.open(ref) + stream = reader.open(dict(ref)) with stream: while total <= size: chunk = stream.read(min(_READ_CHUNK, size + 1 - total)) @@ -125,7 +189,7 @@ def read(ref: Mapping, reader) -> bytes: "ArtifactUnavailable", f"the fixed preparation could not be read: {run_store._describe(exc)}", ) from exc - if total != size or digest.hexdigest() != ref["sha256"]: + if total != size or digest.hexdigest() != expected_sha256: raise _refuse( "ArtifactIntegrityError", "the fixed preparation bytes differ from their ArtifactRef" ) @@ -170,10 +234,7 @@ def project(data: bytes) -> FixedPreparation: for name in ("format_schema", "identity_rule"): if not _is_definition(contract[name]): errors.append(f"prepared.candidate_contract.{name}: not a DefinitionRef") - if not run_store._has_fields(recipe, _RECIPE_FIELDS): - errors.append("prepared.recipe: not the OperationRecipe fields") - elif not _is_sha256(recipe["code_sha256"]): - errors.append("prepared.recipe.code_sha256: not a Sha256") + errors.extend(_recipe_errors(recipe)) if expected is not None: if not run_store._has_fields(expected, _EXPECTED_FIELDS): errors.append( @@ -199,6 +260,7 @@ def project(data: bytes) -> FixedPreparation: identity_rule=contract["identity_rule"], expected_candidate=expected, code_sha256=recipe["code_sha256"], + recipe=recipe, ) @@ -224,6 +286,7 @@ def _check_declarations(spec: Mapping, fixed: FixedPreparation) -> None: for index, step in enumerate(spec["steps"]) if step["expected_code_sha256"] != fixed.code_sha256 ] + errors.extend(_worker_errors(spec, fixed.recipe)) candidate = spec["candidate"] if candidate["input_role"] != fixed.candidate_input_role: errors.append("candidate.input_role: differs from the frozen candidate contract") @@ -253,6 +316,37 @@ def _check_declarations(spec: Mapping, fixed: FixedPreparation) -> None: ) +def _worker_errors(spec: Mapping, recipe: dict) -> list[str]: + """Preserve the frozen worker's entry, paths and placeholder-expanded arguments.""" + errors = [] + if len(spec["steps"]) != 1: + return ["steps: the frozen recipe binds exactly one worker step"] + step = spec["steps"][0] + for name in ("runtime_role", "entrypoint_kind", "entrypoint", "entrypoint_revision", "outputs"): + if step[name] != recipe[name]: + errors.append(f"steps[0].{name}: differs from the frozen worker declaration") + if step["cwd"] != ".": + errors.append("steps[0].cwd: frozen recipe paths are relative to the attempt directory") + if not set(recipe["required_capabilities"]) <= set(spec["required_capabilities"]): + errors.append("required_capabilities: missing a frozen worker requirement") + inputs = {item["role"]: item["materialized_path"] for item in spec["inputs"]} + if not set(recipe["input_roles"]) <= inputs.keys(): + errors.append("inputs: missing a frozen worker input role") + else: + outputs = {item["role"]: item["path"] for item in recipe["outputs"]} + paths = {"input": inputs, "output": outputs} + args = [] + for arg in recipe["args"]: + match = _PLACEHOLDER.fullmatch(arg) + args.append(paths[match[1]][match[2]] if match else arg) + if step["args"] != args: + errors.append("steps[0].args: differs from the frozen recipe binding") + ordered = sorted(tuple(path.split("/")) for path in [*inputs.values(), *outputs.values()]) + if any(b[:len(a)] == a for a, b in zip(ordered, ordered[1:])): + errors.append("inputs/outputs: materialized paths must not overlap") + return errors + + def verify(spec: Mapping, reader) -> FixedPreparation: """Check an evaluation-mode spec against its fixed preparation before registration. diff --git a/tests/check_installed.py b/tests/check_installed.py index 5b02b0d9f..2279c1e1b 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -27,6 +27,15 @@ from inferrsi.resources import repository_packs from inferrsi.resources.repository_packs import REPO_TYPES, repository_pack +# The new Harness binding port ships independently of Apps and Benchmark. +subprocess.run([ + sys.executable, "-I", "-B", "-c", + "import sys; from inferrsi.harness import prepared_binding as p; " + "assert all(callable(getattr(p, n)) for n in ('read', 'project', 'verify')); " + "assert not any(n == root or n.startswith(root + '.') for n in sys.modules " + "for root in ('inferrsi.apps', 'inferrsi.benchmark', 'inferrsi.knowledge'))", +], check=True, capture_output=True, text=True) + assert Path(inferrsi.__file__).resolve().parent.parent.name != "src", ( "This check requires an installed package" ) diff --git a/tests/test_harness_prepared_binding.py b/tests/test_harness_prepared_binding.py index 4601ebac3..fb16b9e9a 100644 --- a/tests/test_harness_prepared_binding.py +++ b/tests/test_harness_prepared_binding.py @@ -63,6 +63,7 @@ def forbidden(*args, **kwargs): monkeypatch.setattr(store, "claim", forbidden) monkeypatch.setattr(execution_service.local_process, "execute_command", forbidden) + monkeypatch.setattr(execution_service, "_argv", forbidden) return spec, reader, store, harness @@ -114,3 +115,242 @@ def test_corrupt_preparation_bytes_cannot_reach_registration(binding): harness.execute(spec) assert caught.value.code == "ArtifactIntegrityError" assert_empty(store) + + +def replace_prepared(binding, path, value): + spec, reader, _, _ = binding + prepared = decode(reader.data) + parent = prepared + for key in path[:-1]: + parent = parent[key] + parent[path[-1]] = value + reader.data = encode(prepared) + spec["prepared_ref"]["artifact"].update( + sha256=sha256(reader.data).hexdigest(), size_bytes=len(reader.data), + ) + + +@pytest.mark.parametrize("path,value", [ + (("prepared_ref", "prepared_id"), "sha256:" + "0" * 64), + (("operation",), "different"), (("validation_scope",), "different"), +]) +def test_all_preparation_identity_disagreements_precede_registration(binding, path, value): + spec, reader, store, harness = binding + parent = spec + for key in path[:-1]: + parent = parent[key] + parent[path[-1]] = value + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "IdentityMismatch" + assert len(reader.opens) == 1 + assert_empty(store) + + +@pytest.mark.parametrize("field,value", [ + ("runtime_role", "another-python"), ("entrypoint_kind", "executable"), + ("entrypoint", "another_worker"), ("entrypoint_revision", "2"), + ("args", ["inputs/candidate.txt", "outputs/other.json"]), ("cwd", "elsewhere"), + ("outputs", []), +]) +def test_worker_declarations_must_preserve_the_frozen_recipe(binding, field, value): + spec, reader, store, harness = binding + spec["steps"][0][field] = value + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "InvalidExecutionSpec" + assert caught.value.stage == "prepared_binding" + assert len(reader.opens) == 1 + assert_empty(store) + + +@pytest.mark.parametrize("field,value", [ + ("path", "inputs/candidate.txt"), ("required", False), ("role", "other"), + ("media_type", "text/plain"), ("format_schema", {"name": "fixture-grader", "revision": "2"}), +]) +def test_output_relabeling_cannot_change_the_frozen_worker_contract(binding, field, value): + spec, _, store, harness = binding + spec["steps"][0]["outputs"][0][field] = value + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "InvalidExecutionSpec" + assert_empty(store) + + +@pytest.mark.parametrize("failure", ["capability", "missing-role", "overlap", "extra-step"]) +def test_incomplete_or_ambiguous_worker_binding_is_refused(binding, failure): + spec, _, store, harness = binding + if failure == "capability": + spec["required_capabilities"] = [] + elif failure == "missing-role": + replace_prepared(binding, ("recipe", "input_roles"), ["candidate", "config"]) + elif failure == "extra-step": + extra = deepcopy(spec["steps"][0]) + extra["step_id"] = "extra" + spec["steps"].append(extra) + else: + # Keep argument binding internally consistent while overlapping an output. + spec["inputs"][0]["materialized_path"] = "outputs" + spec["steps"][0]["args"][0] = "outputs" + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "InvalidExecutionSpec" + assert_empty(store) + + +@pytest.mark.parametrize("path,value", [ + (("schema_version",), True), (("schema_version",), 2), + (("candidate_contract",), None), (("candidate_contract", "identity_rule"), {}), + (("expected_candidate",), {}), (("expected_candidate", "sha256"), None), + (("recipe",), None), (("recipe", "code_sha256"), None), + (("recipe", "runtime_role"), None), (("recipe", "entrypoint_kind"), "unknown"), + (("recipe", "entrypoint"), None), (("recipe", "entrypoint_revision"), ""), + (("recipe", "args"), [None]), (("recipe", "args"), ["{input:unknown}"]), + (("recipe", "args"), ["prefix{input:candidate}"]), + (("recipe", "input_roles"), ["candidate", "candidate"]), + (("recipe", "required_capabilities"), [None]), + (("recipe", "outputs"), [{}]), (("recipe", "outputs", 0, "path"), "../outside"), + (("recipe", "outputs", 0, "format_schema"), {}), +]) +def test_malformed_fixed_projection_is_refused_with_a_declared_error(binding, path, value): + _, _, store, harness = binding + replace_prepared(binding, path, value) + with pytest.raises(PublicError) as caught: + harness.execute(binding[0]) + assert caught.value.code == ("UnsupportedSchema" if value == 2 else "InvalidExecutionSpec") + assert_empty(store) + + +@pytest.mark.parametrize("revision", ["2", "unknown"]) +def test_unknown_candidate_identity_revision_is_never_downgraded(binding, revision): + spec, _, store, harness = binding + replace_prepared(binding, ("candidate_contract", "identity_rule", "revision"), revision) + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "UnsupportedRuleVersion" + assert_empty(store) + + +@pytest.mark.parametrize("rule", ["captured-sha256", "expected-sha256"]) +def test_expected_hash_requirement_depends_on_the_frozen_identity_rule(binding, rule): + spec, _, store, harness = binding + replace_prepared(binding, ("candidate_contract", "identity_rule", "name"), rule) + replace_prepared(binding, ("expected_candidate",), None) + spec["inputs"][0].update( + expected_sha256=None, source_namespace="controller", source_ref=None, + source="candidate.txt", + ) + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == ("InvalidExecutionSpec" if rule == "expected-sha256" + else "UnsupportedOperation") + assert_empty(store) + + +@pytest.mark.parametrize("mode", ["noncanonical", "duplicate-key", "array", "extra-field"]) +def test_unparseable_or_unknown_preparation_fields_never_register(binding, mode): + spec, reader, store, harness = binding + if mode == "noncanonical": + reader.data += b"\n" + elif mode == "duplicate-key": + reader.data = reader.data[:-1] + b',"schema_version":1}' + elif mode == "array": + reader.data = b"[]" + else: + replace_prepared(binding, ("unknown",), True) + spec["prepared_ref"]["artifact"].update( + sha256=sha256(reader.data).hexdigest(), size_bytes=len(reader.data), + ) + with pytest.raises(PublicError) as caught: + harness.execute(spec) + expected = ("ArtifactIntegrityError" if mode in {"noncanonical", "duplicate-key"} + else "InvalidExecutionSpec") + assert caught.value.code == expected + assert_empty(store) + + +@pytest.mark.parametrize("path,value", [ + (("schema_version",), True), (("schema_version",), 2), (("unknown",), None), + (("prepared_ref",), None), (("prepared_ref", "schema_version"), 2), + (("prepared_ref", "artifact", "location"), "../outside"), + (("prepared_ref", "artifact", "size_bytes"), True), + (("prepared_ref", "artifact", "sha256"), "A" * 64), + (("prepared_ref", "artifact", "media_type"), "text/plain"), + (("protocol_digest",), "invalid"), (("candidate",), {}), + (("candidate", "kind"), None), (("candidate", "parent_id"), "parent"), + (("candidate", "context_refs"), [None]), (("candidate", "input_role"), "absent"), + (("steps", 0, "expected_code_sha256"), "invalid"), +]) +def test_static_shape_errors_never_open_the_preparation(binding, path, value): + spec, reader, store, harness = binding + parent = spec + for key in path[:-1]: + parent = parent[key] + parent[path[-1]] = value + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == ("UnsupportedSchema" if path == ("schema_version",) and value == 2 + else "InvalidExecutionSpec") + assert reader.opens == [] + assert_empty(store) + + +@pytest.mark.parametrize("mode", ["undersize", "oversize", "wrong-hash", "missing", "os-error"]) +def test_fixed_byte_reader_closes_streams_and_bounds_reads(binding, mode): + spec, reader, store, harness = binding + original = spec["prepared_ref"]["artifact"] + sizes = [] + + class Tracking(BytesIO): + def read(self, size=-1): + sizes.append(size) + return super().read(size) + + stream = Tracking(reader.data + (b"x" if mode == "oversize" else b"")) + + class Tracked: + def open(self, ref): + if mode == "os-error": + raise FileNotFoundError("fixture missing") + return stream + + if mode == "undersize": + original["size_bytes"] += 1 + elif mode == "wrong-hash": + original["sha256"] = "0" * 64 + with pytest.raises(PublicError) as caught: + prepared_binding.verify(spec, None if mode == "missing" else Tracked()) + assert caught.value.code == ("ArtifactUnavailable" if mode in {"missing", "os-error"} + else "ArtifactIntegrityError") + if mode not in {"missing", "os-error"}: + assert stream.closed + assert max(sizes) <= original["size_bytes"] + 1 + assert_empty(store) + + +def test_reader_cannot_rewrite_the_ref_it_is_meant_to_verify(binding): + spec, _, store, _ = binding + original = deepcopy(spec["prepared_ref"]["artifact"]) + + class Mutating: + def open(self, ref): + ref.update(sha256=sha256(b"{}").hexdigest(), size_bytes=2) + return BytesIO(b"{}") + + with pytest.raises(PublicError) as caught: + prepared_binding.verify(spec, Mutating()) + assert caught.value.code == "ArtifactIntegrityError" + assert spec["prepared_ref"]["artifact"] == original + assert_empty(store) + + +def test_reader_programming_errors_propagate_without_registration(binding): + spec, _, store, _ = binding + + class Broken: + def open(self, ref): + raise TypeError("broken host reader") + + with pytest.raises(TypeError, match="broken host reader"): + prepared_binding.verify(spec, Broken()) + assert_empty(store) From c8bb5048ce7db9e318505e292c496a805575ba4b Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 20:20:19 +0800 Subject: [PATCH 3/8] feat(harness): capture trusted entry bytes before execution Resolve standalone source entries under an explicit trusted root without imports. Retain the actual bytes and digest and preserve existing snapshots. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- src/inferrsi/harness/entry_code.py | 60 ++++++++++++++++++++++++++++++ tests/test_harness_entry_code.py | 39 +++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 src/inferrsi/harness/entry_code.py create mode 100644 tests/test_harness_entry_code.py diff --git a/src/inferrsi/harness/entry_code.py b/src/inferrsi/harness/entry_code.py new file mode 100644 index 000000000..ef8ed3e6a --- /dev/null +++ b/src/inferrsi/harness/entry_code.py @@ -0,0 +1,60 @@ +"""Capture a trusted local source entry without importing or executing worker code.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +from . import input_binding + +_MODULE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*", re.ASCII) + + +class EntryUnavailable(Exception): + """The supported entry bytes cannot be resolved; no actual digest is known.""" + + def __init__(self, reason: str): + super().__init__(reason) + self.reason = reason + + +class EntryCaptureError(Exception): + """A resolved entry could not be retained safely before execution.""" + + +@dataclass(frozen=True) +class CapturedEntry: + path: Path + sha256: str + size_bytes: int + + +def capture_module(root: Path, module: str, directory: Path) -> CapturedEntry: + """Snapshot one source module from an explicit trusted root. + + The first CPU binding supports a standalone ``.py`` entry. Dotted + package names, bytecode/native modules and import hooks are not resolved by + this port. In particular, neither a package initializer nor a worker module + is imported to locate its code. The caller owns the private capture directory. + """ + if not isinstance(module, str) or _MODULE.fullmatch(module) is None: + raise EntryUnavailable("entry_kind_unsupported") + name = module + ".py" + try: + stream = input_binding.open_regular(root, name) + except (OSError, input_binding.BindingError): + raise EntryUnavailable("entry_bytes_unavailable") from None + destination = directory / name + with stream: + # A capture directory belongs to one registered attempt. Never replace + # an existing entry snapshot on an interrupted or repeated operation. + if destination.exists(): + raise EntryCaptureError("entry snapshot already exists") + try: + digest, size = input_binding.copy_stream(stream, destination) + if input_binding.file_sha256(destination) != (digest, size): + raise EntryCaptureError("entry snapshot differs from the captured bytes") + except OSError as error: + raise EntryCaptureError("entry snapshot could not be retained") from error + return CapturedEntry(destination, digest, size) diff --git a/tests/test_harness_entry_code.py b/tests/test_harness_entry_code.py new file mode 100644 index 000000000..9af659f87 --- /dev/null +++ b/tests/test_harness_entry_code.py @@ -0,0 +1,39 @@ +"""Actual entry-byte capture is separate from preparation declaration checks.""" + +from hashlib import sha256 + +import pytest + +from inferrsi.harness import entry_code + + +def test_capture_reads_actual_source_without_importing_or_executing_it(tmp_path): + root = tmp_path / "trusted" + root.mkdir() + source = b"raise RuntimeError('this worker must never be imported during binding')\n" + (root / "fixture_worker.py").write_bytes(source) + result = entry_code.capture_module(root, "fixture_worker", tmp_path / "captures") + assert result.path.read_bytes() == source + assert (result.sha256, result.size_bytes) == (sha256(source).hexdigest(), len(source)) + (root / "fixture_worker.py").write_bytes(b"changed after capture\n") + assert result.path.read_bytes() == source + + +@pytest.mark.parametrize("module", ["missing", "package.worker", "../outside", "worker.py"]) +def test_unresolvable_entry_has_no_actual_digest_or_capture(tmp_path, module): + with pytest.raises(entry_code.EntryUnavailable): + entry_code.capture_module(tmp_path, module, tmp_path / "captures") + assert not (tmp_path / "captures").exists() + + +def test_existing_snapshot_is_preserved_instead_of_replaced(tmp_path): + root = tmp_path / "trusted" + root.mkdir() + (root / "worker.py").write_bytes(b"new bytes") + captures = tmp_path / "captures" + captures.mkdir() + retained = captures / "worker.py" + retained.write_bytes(b"already retained") + with pytest.raises(entry_code.EntryCaptureError): + entry_code.capture_module(root, "worker", captures) + assert retained.read_bytes() == b"already retained" From 2cf33879b72b7bead9f2ea6c5d7b4b67db8bf467 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 20:31:37 +0800 Subject: [PATCH 4/8] feat(harness): commit observed entry verification before worker launch Capture the configured standalone source entry after registration, retain observed code identity, and commit fixed zero-start refusals for missing or mismatching bytes. Execute matching entries from their retained source snapshot. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/apps-module-migrations.json | 11 +- docs/contracts/harness-entry-verification.md | 58 +++++ docs/contracts/harness-prepared-binding.md | 11 +- docs/operations.md | 1 + src/inferrsi/harness/api.py | 2 +- src/inferrsi/harness/execution_service.py | 108 +++++++-- tests/check_installed.py | 1 + tests/test_harness_entry_verification.py | 227 +++++++++++++++++++ tests/test_harness_execute.py | 1 + 9 files changed, 388 insertions(+), 32 deletions(-) create mode 100644 docs/contracts/harness-entry-verification.md create mode 100644 tests/test_harness_entry_verification.py diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index a74372bf8..3916e8de8 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -22,12 +22,12 @@ "docs/operations.md", "src/inferrsi/apps/api.py", "src/inferrsi/apps/archive.py", - "src/inferrsi/apps/workflows/benchmark_compare.py", "src/inferrsi/apps/delivery/__init__.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", "src/inferrsi/apps/storage/artifact_reader.py", "src/inferrsi/apps/views/attempt.py", + "src/inferrsi/apps/workflows/benchmark_compare.py", "tests/check_installed.py", "tests/test_apps_assess.py", "tests/test_apps_module_migrations.py", @@ -37,6 +37,7 @@ "tests/test_apps_publication.py", "tests/test_apps_replay.py", "tests/test_benchmark_preparation.py", + "tests/test_harness_entry_verification.py", "tests/test_result_delivery.py" ] }, @@ -60,7 +61,8 @@ "tests/test_apps_artifact_reader.py", "tests/test_apps_assess.py", "tests/test_apps_generation_plan.py", - "tests/test_apps_module_migrations.py" + "tests/test_apps_module_migrations.py", + "tests/test_harness_entry_verification.py" ] }, { @@ -127,7 +129,6 @@ ], "consumers": [ "docs/apps-output.md", - "tests/test_harness_prepared_binding.py", "docs/architecture.md", "docs/module-layout.md", "docs/operations.md", @@ -140,7 +141,9 @@ "tests/test_apps_execution_binding.py", "tests/test_apps_generation_plan.py", "tests/test_apps_module_migrations.py", - "tests/test_benchmark_preparation.py" + "tests/test_benchmark_preparation.py", + "tests/test_harness_entry_verification.py", + "tests/test_harness_prepared_binding.py" ] }, { diff --git a/docs/contracts/harness-entry-verification.md b/docs/contracts/harness-entry-verification.md new file mode 100644 index 000000000..1dba30e76 --- /dev/null +++ b/docs/contracts/harness-entry-verification.md @@ -0,0 +1,58 @@ +# Harness source-entry verification + +The local CPU target can opt into the first frozen evaluation task with +`LocalCpuTarget(..., module_root=)`. The host owns +this configuration. Requests cannot supply the root. Existing targets with no +module root retain their ordinary-command behavior and refuse evaluation mode +before registration. + +The first target supports a standalone source entry named +`module_root/.py` with `entrypoint_kind="python_module"`. It reads a +regular file inside that root, without importing worker code or package +initializers. Dotted packages, bytecode/native modules and native executables +remain unsupported on this evaluation target. No ambient module/path fallback +is attempted. Ordinary-command module/executable resolution is unchanged. + +## Ordering and retained facts + +1. Static shape and fixed-preparation declaration checks run before registration. + Malformed bindings still produce no attempt or record. +2. RunStore registers the request and ordinary admission runs. +3. Harness copies the actual entry into its private attempt capture directory, + independently checks the retained bytes, and compares their SHA-256 with the + expected digest already bound to the frozen preparation. +4. A mismatch yields a fixed `not_started` record. Its + `entry_code_identity` check is `failed`, the runtime binding contains the + observed actual digest, and the check references the retained source bytes. +5. An unavailable/unsupported entry yields a fixed `not_started` record with a + `not_performed` check and no source reference. A source-module binding keeps + the actual digest null and the unresolved argv empty. No mismatch is invented. +6. A matching standalone module executes from its captured source file using the + current Python interpreter with `-I -B`. The workspace and `PYTHONPATH` cannot + replace the bound entry, and subsequent changes to its original source file + do not change the bytes being executed. Package execution is outside this + target's supported range. The runtime binding records the actual command. + +Both refusal cases launch zero worker processes, acquire no process lease, and +retain the preparation/protocol/candidate identity in the fixed record. The +source proof uses `python-source@1`; its SHA is the observed byte identity, not +the frozen expectation copied into an observation. The complete interpreter is +not archived. Re-reading the fixed revision preserves those facts. + +The entry capture is a Harness write. A capture failure uses the existing +`ExecutionPersistenceError` path with `started=false`. Record commit failure +also preserves that known start state and cannot produce a committed RecordRef. +There is no process fallback after an entry failure. + +## Delivery boundary + +These checks implement the post-registration T37-N1/N2 boundary separately from +the pre-registration binding tests. A real matching source entry is a positive +control. CPU execution facts and exit zero are not a Benchmark verdict. + +The production preparation/Apps archive cross-tests use the accepted D023 / +`t108-v4` family and actual source bytes. Their small software worker does not +claim complete F/C/M evidence, a service gain, or whole-Issue production +acceptance. Existing ordinary CPU supervision is reused; evaluation deadline, +budget and T40 acceptance remain the later Harness unit, including its separate +launch-boundary prerequisite. diff --git a/docs/contracts/harness-prepared-binding.md b/docs/contracts/harness-prepared-binding.md index 15db9301e..0f4d75804 100644 --- a/docs/contracts/harness-prepared-binding.md +++ b/docs/contracts/harness-prepared-binding.md @@ -45,8 +45,9 @@ and a null step timeout without rewriting historical fixtures. Its canonical bytes must agree between the Apps binder and Harness `request_digest`. The vector is documentary; its historical code pin is not an available worker. -This slice keeps valid evaluation-mode execution explicitly unsupported. -Actual entry resolution and code-byte verification after registration (T37), -attempt deadlines/budget enforcement with a null step timeout, and evaluation -replay (T40) are later delivery slices. It does not establish production CPU -acceptance or completion of the governing Harness or Apps Issues. +Targets without an explicit `module_root` keep valid evaluation-mode execution +unsupported before registration. A configured source target performs the separate +[post-registration entry check](harness-entry-verification.md) (T37). Attempt +deadlines/budget acceptance with a null step timeout and evaluation replay (T40) +remain separate delivery work. Declaration checks alone do not establish +production CPU acceptance or completion of the governing Harness or Apps Issues. diff --git a/docs/operations.md b/docs/operations.md index 4e4531453..1868dfb2c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -53,6 +53,7 @@ Effect classes used below: | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | | Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) | +| Verify and execute one frozen source evaluation entry | Python `inferrsi.harness.api.Harness.execute` with trusted `LocalCpuTarget(module_root=...)` | Experiment-executing / record-writing | Fixed preparation and expected code digest; actual source captured after registration; T37 refusals start no worker; [source-entry contract](contracts/harness-entry-verification.md) | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | diff --git a/src/inferrsi/harness/api.py b/src/inferrsi/harness/api.py index cc69e8f5a..3643dc7f0 100644 --- a/src/inferrsi/harness/api.py +++ b/src/inferrsi/harness/api.py @@ -43,7 +43,7 @@ def allocate( ) def execute(self, spec: Mapping, *, cancellation=None) -> dict: - """The flat ExecutionRecord of one ordinary command request, or a PublicError. + """The flat ExecutionRecord of one supported local CPU request, or a PublicError. ``cancellation`` is a host handle with ``is_set()``, such as ``threading.Event``. """ diff --git a/src/inferrsi/harness/execution_service.py b/src/inferrsi/harness/execution_service.py index 66995746e..38d756b06 100644 --- a/src/inferrsi/harness/execution_service.py +++ b/src/inferrsi/harness/execution_service.py @@ -12,12 +12,12 @@ from pathlib import Path from inferrsi.contracts import json_v1 -from inferrsi.harness import input_binding, local_process, prepared_binding, run_store +from inferrsi.harness import entry_code, input_binding, local_process, prepared_binding, run_store from inferrsi.harness.errors import PublicError -# H02 single local-CPU execute in ordinary command mode (V05 null set). Evaluation-mode -# specs are checked before registration (V05/V28/V32 and their fixed preparation) and then -# refused: the T37 entry-code check and generation chaining are later slices. +# The ordinary CPU path preserves V05 null mode. Evaluation requires an explicit trusted +# module root, a fixed preparation binding and an actual source-entry snapshot before start. +# Generation chaining and the remaining evaluation replay/limit acceptance are later slices. ATTEMPT_FIELDS = ( "request_id", "run_id", "attempt_id", "owner", "target_request", "deadline_utc", "resources", "budget", "retry_of", "retry_reason", @@ -57,6 +57,7 @@ _STREAM_MEDIA_TYPE = "application/octet-stream" _INPUT_MEDIA_TYPE = "application/octet-stream" _INPUT_SCHEMA = {"name": "input-identity", "revision": "1"} +_ENTRY_SCHEMA = {"name": "python-source", "revision": "1"} # The one snapshot format this slice captures: the bytes of one regular file. SNAPSHOT_FORMAT = "file-v1" # The only capability this target provides: one command tree in an OS-owned domain. @@ -73,7 +74,10 @@ class LocalCpuTarget: ``work_root`` holds per-attempt workspaces (``workspaces/``) and private capture snapshots. ``controller_root`` / ``target_root`` authorize input sources of - those namespaces; ``None`` leaves that namespace unauthorized. + those namespaces; ``None`` leaves that namespace unauthorized. ``module_root`` + explicitly enables evaluation through standalone source entries under that trusted + root. The retained source is executed directly in an isolated Python interpreter; + package imports and ambient cwd/PYTHONPATH entry resolution are not used. """ target_id: str @@ -81,6 +85,7 @@ class LocalCpuTarget: work_root: Path controller_root: Path | None = None target_root: Path | None = None + module_root: Path | None = None def __post_init__(self) -> None: errors = [ @@ -88,7 +93,7 @@ def __post_init__(self) -> None: for name in ("target_id", "config_revision") if not run_store._is_id(getattr(self, name)) ] - for name in ("work_root", "controller_root", "target_root"): + for name in ("work_root", "controller_root", "target_root", "module_root"): value = getattr(self, name) if value is None: continue @@ -403,10 +408,8 @@ def validate(spec: object, target: LocalCpuTarget, *, artifact_reader=None) -> N # request. The reader is the read-only port; H does not write or register anything. prepared_binding.verify(spec, artifact_reader) unsupported = [] - if evaluation: - # The T37 pre-start entry-code check runs after registration and is a later slice. - # Until it exists, no evaluation-mode candidate may start. - unsupported.append("evaluation-mode entry verification is not implemented in this slice") + if evaluation and target.module_root is None: + unsupported.append("this target has no configured evaluation module root") if spec["generation_record_ref"] is not None: unsupported.append("generation chaining is not implemented in this slice") request = spec["target_request"] @@ -578,8 +581,8 @@ def _capture_input(ctx: _Context, index: int, item: Mapping, target: LocalCpuTar "checker_revision": "1", "subject": item["role"], "phase": "preparing", - "operation": None, - "validation_scope": None, + "operation": ctx.spec["operation"], + "validation_scope": ctx.spec["validation_scope"], "attempt_id": ctx.attempt_id, "status": {"matched": "passed", "mismatched": "failed", "not_performed": "not_performed", "error": "execution_error"}[status], @@ -600,6 +603,48 @@ def _argv(step: Mapping) -> list[str] | None: return None if executable is None else [executable, *step["args"]] +def _bind_entry(ctx: _Context, step: Mapping, target: LocalCpuTarget): + """Observe/capture the supported entry after registration, before workspace execution.""" + actual, ref, argv = None, None, [] + reason = "entry_kind_unsupported" + if step["entrypoint_kind"] == "python_module": + try: + captured = entry_code.capture_module( + target.module_root, step["entrypoint"], ctx.captures / "entry", + ) + except entry_code.EntryUnavailable as error: + reason = error.reason + except entry_code.EntryCaptureError as error: + raise _RetentionFailure("entry capture failed") from error + else: + actual = captured.sha256 + ref = ctx.object_ref(captured.path, actual, captured.size_bytes, "text/x-python") + # Execute exactly the retained source, with no cwd/PYTHONPATH module fallback. + # This first target supports standalone source modules, not Python packages. + argv = [sys.executable, "-I", "-B", str(captured.path), *step["args"]] + reason = ("entry_code_matched" if actual == step["expected_code_sha256"] + else "entry_code_sha256_differs") + status = ("not_performed" if actual is None else + "passed" if actual == step["expected_code_sha256"] else "failed") + runtime = { + "step_id": step["step_id"], "runtime_role": step["runtime_role"], + "executable": sys.executable, "entrypoint": step["entrypoint"], + "entrypoint_revision": step["entrypoint_revision"], "code_sha256": actual, + "argv": argv, "environment_identity": None, + } if step["entrypoint_kind"] == "python_module" else None + check = { + "check_id": "entry_code_identity", "checker_revision": "1", "subject": step["step_id"], + "phase": "preparing", "operation": ctx.spec["operation"], + "validation_scope": ctx.spec["validation_scope"], "attempt_id": ctx.attempt_id, + "status": status, "producer": { + "kind": "harness_capture", "identity": _HARNESS, "revision": "1", + "control_domain": _CONTROL_DOMAIN, "capture_route": "entry-source-snapshot", + }, "evidence_refs": [] if ref is None else [ref], "artifact_schema": _ENTRY_SCHEMA, + "reasons": [reason], + } + return argv, runtime, check + + def _retain_stream(ctx: _Context, stream, name: str, media_type: str) -> dict | None: """Snapshot one produced file from its open stream; None when the copy fails.""" target = ctx.captures / name @@ -848,12 +893,12 @@ def _payload(ctx: _Context, **facts) -> dict: "target_binding": facts["target_binding"], "retry_of": spec["retry_of"], "retry_reason": spec["retry_reason"], - "generation_record_ref": None, - "prepared_ref": None, - "protocol_digest": None, - "operation": None, - "validation_scope": None, - "candidate": None, + "generation_record_ref": copy.deepcopy(spec["generation_record_ref"]), + "prepared_ref": copy.deepcopy(spec["prepared_ref"]), + "protocol_digest": spec["protocol_digest"], + "operation": spec["operation"], + "validation_scope": spec["validation_scope"], + "candidate": copy.deepcopy(spec["candidate"]), "record_version": 1, "recorded_at": run_store.utc_now(), "bound_inputs": facts["bound_inputs"], @@ -907,7 +952,22 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: binding = _target_binding(target, capability_checks, []) if errors: return _refusal(ctx, digest, binding, admission, [], [], errors, "admission_rejected") - argvs = [_argv(step) for step in ctx.spec["steps"]] + evaluation = ctx.spec["prepared_ref"] is not None + entry_checks = [] + if evaluation: + entries = [_bind_entry(ctx, step, target) for step in ctx.spec["steps"]] + argvs = [entry[0] for entry in entries] + binding["runtime_bindings_by_step"] = [ + entry[1] for entry in entries if entry[1] is not None + ] + entry_checks = [entry[2] for entry in entries] + if any(check["status"] != "passed" for check in entry_checks): + code = ("entry_code_mismatch" if any(check["status"] == "failed" + for check in entry_checks) else "entrypoint_unavailable") + return _refusal(ctx, digest, binding, admission, [], entry_checks, + [_error(code, "binding", "evaluation entry binding refused")], code) + else: + argvs = [_argv(step) for step in ctx.spec["steps"]] if None in argvs: errors = [_error("entrypoint_unavailable", "binding", "an executable entry is not found")] return _refusal(ctx, digest, binding, admission, [], [], errors, "entrypoint_unavailable") @@ -915,7 +975,9 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: ctx.workspace.mkdir(parents=True) except FileExistsError: errors = [_error("workspace_exists", "binding", "the attempt workspace already exists")] - return _refusal(ctx, digest, binding, admission, [], [], errors, "workspace_exists") + return _refusal( + ctx, digest, binding, admission, [], entry_checks, errors, "workspace_exists", + ) except OSError as exc: raise _RetentionFailure("workspace creation failed") from exc captured = [ @@ -923,7 +985,7 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: for index, item in enumerate(ctx.spec["inputs"]) ] bound = [item[0] for item in captured] - checks = [item[1] for item in captured] + checks = entry_checks + [item[1] for item in captured] failed = [b for b in bound if b["identity_status"] in ("mismatched", "error")] if failed: # H02-3: the actual binding is refused after registration and before any start. @@ -946,7 +1008,7 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: outcome, exit_code = _outcome(steps) cleanup, leases = _cleanup(runs) launched = {step["step_id"] for step, run in runs if run["pid"] is not None} - binding["runtime_bindings_by_step"] = [ + ordinary_runtime = [ { "step_id": step["step_id"], "runtime_role": step["runtime_role"], @@ -961,6 +1023,8 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: for step, argv in zip(ctx.spec["steps"], argvs) if step["step_id"] in launched ] + if not evaluation: + binding["runtime_bindings_by_step"] = ordinary_runtime codes = {"timed_out": "timeout", "cancelled": "cancelled", "execution_error": "execution_error", "not_started": "deadline_exhausted"} run_errors = ( diff --git a/tests/check_installed.py b/tests/check_installed.py index 2279c1e1b..f6da835c0 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -31,6 +31,7 @@ subprocess.run([ sys.executable, "-I", "-B", "-c", "import sys; from inferrsi.harness import prepared_binding as p; " + "from inferrsi.harness import entry_code; assert callable(entry_code.capture_module); " "assert all(callable(getattr(p, n)) for n in ('read', 'project', 'verify')); " "assert not any(n == root or n.startswith(root + '.') for n in sys.modules " "for root in ('inferrsi.apps', 'inferrsi.benchmark', 'inferrsi.knowledge'))", diff --git a/tests/test_harness_entry_verification.py b/tests/test_harness_entry_verification.py new file mode 100644 index 000000000..b392287a2 --- /dev/null +++ b/tests/test_harness_entry_verification.py @@ -0,0 +1,227 @@ +"""T37 happens after registration; declaration-only refusals remain in their own test file.""" + +import json +import sqlite3 +import subprocess +import sys +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from inferrsi.apps import api +from inferrsi.apps.handoff.bridge import bind_execution +from inferrsi.apps.storage.archive import read_prepared +from inferrsi.apps.storage.artifact_reader import ArtifactReader +from inferrsi.benchmark import preparation as benchmark +from inferrsi.contracts.json_v1 import decode +from inferrsi.harness import execution_service, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from inferrsi.harness.errors import PublicError +from tests.test_benchmark_preparation import preparation as preparation + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def evaluation(preparation, tmp_path): + service, request, sources, _, _, config, _ = preparation + output = api.prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert output.result["exit_code"] == 0 + ref = output.result["prepared_ref"] + prepared = read_prepared(config, ref) + vector = decode((ROOT / "tests/fixtures/harness-prepared-binding/d023-null-timeout.json") + .read_bytes()) + controller, modules = tmp_path / "controller", tmp_path / "modules" + controller.mkdir() + modules.mkdir() + source = sources["worker-code"].read_bytes() + (modules / "fixture_worker.py").write_bytes(source) + candidate = b"raise RuntimeError('candidate input must never replace the trusted worker')\n" + (controller / "candidate.txt").write_bytes(candidate) + proposal = {key: deepcopy(vector[key]) + for key in ("candidate", "inputs", "generation_record_ref")} + proposal["inputs"][0].update( + source="candidate.txt", source_namespace="controller", source_ref=None, + expected_sha256=sha256(candidate).hexdigest(), + ) + request = { + "schema_version": 1, "prepared_ref": ref, + "attempt": {key: vector[key] for key in execution_service.ATTEMPT_FIELDS}, + "proposal": proposal, "output_root": vector["output_root"], + } + spec = bind_execution(request, prepared) + store = run_store.RunStore(tmp_path / "store", "test") + target = LocalCpuTarget( + "fixture-cpu", "1", work_root=tmp_path / "work", controller_root=controller, + module_root=modules, + ) + reader = ArtifactReader(config) + return {"spec": spec, "store": store, "target": target, "modules": modules, + "source": source, "reader": reader, "harness": Harness(store, target, + artifact_reader=reader)} + + +def assert_fixed_refusal(case, record, status, actual): + schema = json.loads((ROOT / "docs/contracts/schemas/contract-v1.schema.json") + .read_text(encoding="utf-8")) + Draft202012Validator({**schema, "$ref": "#/$defs/ExecutionRecord"}).validate(record) + assert record["started"] is False and record["phase"] == "finished" + assert record["process_outcome"] == "not_started" and record["exit_code"] is None + assert record["prepared_ref"] == case["spec"]["prepared_ref"] + assert record["candidate"] == case["spec"]["candidate"] + check = record["checks"][0] + assert check["check_id"] == "entry_code_identity" and check["status"] == status + assert check["operation"] == "evaluate" and check["validation_scope"] == "p1" + runtime = record["target_binding"]["runtime_bindings_by_step"][0] + assert runtime["code_sha256"] == actual + assert record["resource_leases"]["leases"] == [] + saved = case["store"].read(record["attempt_id"], record["record_version"]) + assert saved["record_ref"] == record["record_ref"] + assert saved["checks"] == record["checks"] + with sqlite3.connect(case["store"].root / run_store.INDEX_NAME) as connection: + assert connection.execute("SELECT count(*) FROM attempts").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM records").fetchone()[0] == 1 + + +@pytest.mark.parametrize("failure", ["mismatch", "missing"]) +def test_t37_entry_failure_commits_a_not_started_record(evaluation, monkeypatch, failure): + case = evaluation + source = case["modules"] / "fixture_worker.py" + if failure == "mismatch": + observed = b"raise RuntimeError('changed worker must not start')\n" + source.write_bytes(observed) + actual = sha256(observed).hexdigest() + else: + source.unlink() + actual = None + + def forbidden(*args, **kwargs): + pytest.fail("T37-N1/N2 launched a process") + + monkeypatch.setattr(execution_service.local_process, "execute_command", forbidden) + record = case["harness"].execute(case["spec"]) + assert_fixed_refusal( + case, record, "failed" if failure == "mismatch" else "not_performed", actual, + ) + refs = record["checks"][0]["evidence_refs"] + if failure == "mismatch": + assert len(refs) == 1 and refs[0]["sha256"] == actual + with case["reader"].open(refs[0]) as stream: + assert stream.read() == observed + else: + assert refs == [] + assert record["target_binding"]["runtime_bindings_by_step"][0]["argv"] == [] + + +@pytest.mark.parametrize("interference", ["workspace-shadow", "changed-origin"]) +def test_matching_entry_executes_its_snapshot_and_retains_the_observed_digest( + evaluation, monkeypatch, interference, +): + case = evaluation + if interference == "workspace-shadow": + case["spec"]["inputs"][0]["materialized_path"] = "fixture_worker.py" + case["spec"]["steps"][0]["args"][0] = "fixture_worker.py" + else: + original = execution_service.entry_code.capture_module + + def capture_then_change(*args, **kwargs): + result = original(*args, **kwargs) + (case["modules"] / "fixture_worker.py").write_bytes( + b"raise RuntimeError('mutable origin is not the bound entry')\n", + ) + return result + + monkeypatch.setattr(execution_service.entry_code, "capture_module", capture_then_change) + record = case["harness"].execute(case["spec"]) + assert record["started"] is True and record["process_outcome"] == "exited" + assert record["exit_code"] == 0 + check = record["checks"][0] + assert check["status"] == "passed" + actual = sha256(case["source"]).hexdigest() + assert record["target_binding"]["runtime_bindings_by_step"][0]["code_sha256"] == actual + assert record["target_binding"]["runtime_bindings_by_step"][0]["argv"][1:3] == ["-I", "-B"] + with case["reader"].open(check["evidence_refs"][0]) as stream: + assert stream.read() == case["source"] + assert "verdict" not in record + + +@pytest.mark.parametrize("failure", ["mismatch", "missing"]) +def test_t37_commit_failure_preserves_not_started_and_no_committed_record( + evaluation, monkeypatch, failure, +): + case = evaluation + source = case["modules"] / "fixture_worker.py" + if failure == "missing": + source.unlink() + else: + source.write_bytes(b"changed worker\n") + + def fail_commit(*args, **kwargs): + raise sqlite3.OperationalError("fixture commit failure") + + monkeypatch.setattr(case["store"], "_append", fail_commit) + monkeypatch.setattr(execution_service.local_process, "execute_command", + lambda *a, **k: pytest.fail("failed T37 commit started a process")) + with pytest.raises(PublicError) as caught: + case["harness"].execute(case["spec"]) + assert caught.value.code == "ExecutionPersistenceError" + assert caught.value.started is False + assert caught.value.latest_record_ref is None + assert caught.value.partial_record is not None + assert caught.value.partial_record.started is False + with pytest.raises(PublicError) as missing: + case["store"].read(case["spec"]["attempt_id"]) + assert missing.value.code == "NoCommittedRecord" + + +def test_t37_fixed_mismatch_and_source_bytes_can_be_read_in_a_fresh_process( + evaluation, monkeypatch, +): + case = evaluation + changed = b"raise RuntimeError('mismatching worker never executes')\n" + source = case["modules"] / "fixture_worker.py" + source.write_bytes(changed) + monkeypatch.setattr(execution_service.local_process, "execute_command", + lambda *a, **k: pytest.fail("mismatching worker started")) + record = case["harness"].execute(case["spec"]) + source.unlink() + script = ( + "import sys,json,hashlib; from inferrsi.harness.run_store import RunStore,RunStoreReader; " + "s=RunStore(sys.argv[1],'test'); r=s.read(sys.argv[2],1); " + "ref=r['checks'][0]['evidence_refs'][0]; " + "stream=RunStoreReader(s.root,'test').open(ref); raw=stream.read(); stream.close(); " + "print(json.dumps([r['record_ref'],r['started'],r['checks'][0]['status']," + "r['target_binding']['runtime_bindings_by_step'][0]['code_sha256']," + "hashlib.sha256(raw).hexdigest()]))" + ) + child = subprocess.run( + [sys.executable, "-c", script, str(case["store"].root), record["attempt_id"]], + check=True, capture_output=True, text=True, timeout=30, + ) + actual = sha256(changed).hexdigest() + assert json.loads(child.stdout) == [record["record_ref"], False, "failed", actual, actual] + + +def test_entry_capture_write_failure_is_not_an_unresolvable_entry(evaluation, monkeypatch): + case = evaluation + + def fail_capture(*args, **kwargs): + raise execution_service.entry_code.EntryCaptureError("fixture capture failure") + + monkeypatch.setattr(execution_service.entry_code, "capture_module", fail_capture) + monkeypatch.setattr(execution_service.local_process, "execute_command", + lambda *a, **k: pytest.fail("failed capture started a worker")) + with pytest.raises(PublicError) as caught: + case["harness"].execute(case["spec"]) + assert caught.value.code == "ExecutionPersistenceError" + assert caught.value.started is False and caught.value.latest_record_ref is None + with pytest.raises(PublicError) as missing: + case["store"].read(case["spec"]["attempt_id"]) + assert missing.value.code == "NoCommittedRecord" diff --git a/tests/test_harness_execute.py b/tests/test_harness_execute.py index 0778e0d05..94e01b964 100644 --- a/tests/test_harness_execute.py +++ b/tests/test_harness_execute.py @@ -38,6 +38,7 @@ "inferrsi.harness.api", "inferrsi.harness.execution_service", "inferrsi.harness.input_binding", "inferrsi.harness.local_process", "inferrsi.harness.prepared_binding", + "inferrsi.harness.entry_code", "inferrsi.harness.atomic_json", "inferrsi.harness.run_store", "inferrsi.harness.errors", "inferrsi.harness.execution_spec", "inferrsi.contracts.json_v1", From 581ef719033dcde3f1a2e1a8675eb308d7959e8c Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 20:40:02 +0800 Subject: [PATCH 5/8] docs(harness): distinguish resolved entry commands from launches Clarify mismatch argv and the single-step source target. Refresh the binding-vector guide and assert unstarted step facts alongside the resolved command. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/harness-entry-verification.md | 9 ++++++++- tests/fixtures/harness-prepared-binding/README.md | 6 ++++-- tests/test_harness_entry_verification.py | 3 +++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/contracts/harness-entry-verification.md b/docs/contracts/harness-entry-verification.md index 1dba30e76..5d3fe1993 100644 --- a/docs/contracts/harness-entry-verification.md +++ b/docs/contracts/harness-entry-verification.md @@ -31,7 +31,14 @@ is attempted. Ordinary-command module/executable resolution is unchanged. current Python interpreter with `-I -B`. The workspace and `PYTHONPATH` cannot replace the bound entry, and subsequent changes to its original source file do not change the bytes being executed. Package execution is outside this - target's supported range. The runtime binding records the actual command. + target's supported range. The runtime binding records the resolved command. + +Runtime bindings describe resolution, not proof of a process launch. On an N1 +mismatch, `argv` is the would-be command over the captured source and the step +remains `not_started`; `started` and step facts determine whether anything ran. +On N2 it is empty because the source command could not be resolved. This target +intentionally binds the frozen recipe's single worker step; declaration checks +reject multiple steps, so each private entry directory holds one snapshot. Both refusal cases launch zero worker processes, acquire no process lease, and retain the preparation/protocol/candidate identity in the fixed record. The diff --git a/tests/fixtures/harness-prepared-binding/README.md b/tests/fixtures/harness-prepared-binding/README.md index e235960c0..5d6e17898 100644 --- a/tests/fixtures/harness-prepared-binding/README.md +++ b/tests/fixtures/harness-prepared-binding/README.md @@ -9,6 +9,8 @@ with request digest This is immutable contract-conformance input, not observed execution evidence or an available worker implementation. Tests compare the canonical Apps binder, -Harness digest and read-only prepared projection. No process is started. T37 -entry verification and production T40 replay remain separate PR-2/PR-3 work. +Harness digest and read-only prepared projection. No process is started by these +binding tests. Post-registration T37 is covered separately in +`tests/test_harness_entry_verification.py`; deadline/budget and production T40 +replay remain PR-3 work. The original documentary fixtures are not rewritten. diff --git a/tests/test_harness_entry_verification.py b/tests/test_harness_entry_verification.py index b392287a2..aa42b0195 100644 --- a/tests/test_harness_entry_verification.py +++ b/tests/test_harness_entry_verification.py @@ -81,6 +81,9 @@ def assert_fixed_refusal(case, record, status, actual): assert check["operation"] == "evaluate" and check["validation_scope"] == "p1" runtime = record["target_binding"]["runtime_bindings_by_step"][0] assert runtime["code_sha256"] == actual + assert all(step["status"] == "not_started" for step in record["steps"]) + if actual is not None: + assert runtime["argv"][1:3] == ["-I", "-B"] assert record["resource_leases"]["leases"] == [] saved = case["store"].read(record["attempt_id"], record["record_version"]) assert saved["record_ref"] == record["record_ref"] From 6bbc55078d5a3b237559818a2d393545001ffc12 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 20:56:06 +0800 Subject: [PATCH 6/8] fix(harness): resolve evaluation reentry before artifact binding Apply existing attempt state and digest conflicts before fresh preparation IO or target support checks. Snapshot the saved request before host callbacks and preserve original records without new registration or execution. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- src/inferrsi/harness/execution_service.py | 30 +++++-- src/inferrsi/harness/run_store.py | 28 ++++++ tests/test_harness_evaluation_replay.py | 101 ++++++++++++++++++++++ 3 files changed, 150 insertions(+), 9 deletions(-) create mode 100644 tests/test_harness_evaluation_replay.py diff --git a/src/inferrsi/harness/execution_service.py b/src/inferrsi/harness/execution_service.py index 38d756b06..5b5942360 100644 --- a/src/inferrsi/harness/execution_service.py +++ b/src/inferrsi/harness/execution_service.py @@ -364,12 +364,8 @@ def _request_errors(spec: Mapping) -> list[str]: return errors -def validate(spec: object, target: LocalCpuTarget, *, artifact_reader=None) -> None: - """Refuse a request before registration (D021 priority: version, fields, support). - - An evaluation-mode spec is also checked against its fixed preparation, read through - ``artifact_reader`` (V05, V28, V32). Nothing here registers an attempt or writes a record. - """ +def _validate_fields(spec: object) -> bool: + """Check static fields before querying existing request state or opening artifacts.""" if isinstance(spec, Mapping): version = spec.get("schema_version") if type(version) is int and version != 1: @@ -403,6 +399,11 @@ def validate(spec: object, target: LocalCpuTarget, *, artifact_reader=None) -> N "InvalidExecutionSpec", stage="validate", message="illegal ExecutionSpec", field_errors=errors, ) + return bool(evaluation) + + +def _validate_fresh(spec, target, artifact_reader, evaluation): + """Validate fixed bytes and target support only for a request not yet registered.""" if evaluation: # Malformed or inconsistent bindings are refused here, before the RunStore sees the # request. The reader is the read-only port; H does not write or register anything. @@ -423,6 +424,12 @@ def validate(spec: object, target: LocalCpuTarget, *, artifact_reader=None) -> N ) +def validate(spec: object, target: LocalCpuTarget, *, artifact_reader=None) -> None: + """Validate a fresh request before registration; this port has no RunStore state.""" + evaluation = _validate_fields(spec) + _validate_fresh(spec, target, artifact_reader, evaluation) + + def _error(code: str, stage: str, message: str) -> dict: return {"code": code, "stage": stage, "message": message, "retryable": False} @@ -1051,11 +1058,16 @@ def execute( cancellation=None, artifact_reader=None, ) -> dict: - """H ``execute``: register, admit, bind, run and commit one ordinary command request.""" - validate(spec, target, artifact_reader=artifact_reader) + """Execute one request or return its existing state before any new binding or start.""" + evaluation = _validate_fields(spec) + spec = copy.deepcopy(dict(spec)) digest = run_store.request_digest(spec) attempt = {name: copy.deepcopy(spec[name]) for name in ATTEMPT_FIELDS} - # Re-entry is decided by the store before any binding or start (V14 / D021 section 3). + existing = store._reenter_if_registered(spec["attempt_id"], digest) + if existing is not None: + return run_store.flatten(existing, run_store.utc_now()) + _validate_fresh(spec, target, artifact_reader, evaluation) + # Another caller may have registered while the fixed preparation was read. claimed = store.claim(attempt, digest) if isinstance(claimed, run_store.CommittedRecord): return run_store.flatten(claimed, run_store.utc_now()) diff --git a/src/inferrsi/harness/run_store.py b/src/inferrsi/harness/run_store.py index 7e6d3e8f9..1d2e7b97e 100644 --- a/src/inferrsi/harness/run_store.py +++ b/src/inferrsi/harness/run_store.py @@ -994,6 +994,34 @@ def _seal_generation_input( ) from exc return values + def _reenter_if_registered( + self, attempt_id: str, request_digest: str, + ) -> CommittedRecord | None: + """Apply existing request state without registering or refreshing an attempt. + + Harness has checked the request's static shape and computed this digest. + A missing row permits fresh binding validation; ``claim`` still arbitrates + a concurrent registration after that validation finishes. + """ + lock_path = self.root / "locks" / f"{attempt_hash(attempt_id)}.lock" + try: + with self._connection() as conn, _transaction(conn): + row = conn.execute( + "SELECT * FROM attempts WHERE request_id = ?", (attempt_id,), + ).fetchone() + return None if row is None else self._reenter( + conn, row, request_digest, lock_path, + ) + except _IntegrityFault as error: + raise PublicError( + "RecordIntegrityError", stage="claim", attempt_id=attempt_id, message=str(error), + ) from error + except (sqlite3.Error, OSError, _StorageFault) as error: + raise PublicError( + "ExecutionPersistenceError", stage="claim", attempt_id=attempt_id, + started=None, message=f"request-state lookup failed: {_describe(error)}", + ) from error + def claim(self, attempt: Mapping, request_digest: str) -> Claim | CommittedRecord: """Register an attempt atomically, or apply the D021 re-entry table to its registration. diff --git a/tests/test_harness_evaluation_replay.py b/tests/test_harness_evaluation_replay.py new file mode 100644 index 000000000..aae8a5616 --- /dev/null +++ b/tests/test_harness_evaluation_replay.py @@ -0,0 +1,101 @@ +"""Evaluation reentry is decided before preparation, target binding or a new launch.""" + +import sqlite3 +from copy import deepcopy + +import pytest + +from inferrsi.harness import execution_service, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from inferrsi.harness.errors import PublicError +from tests.test_benchmark_preparation import preparation as preparation +from tests.test_harness_entry_verification import evaluation as evaluation + + +def counts(case): + with sqlite3.connect(case["store"].root / run_store.INDEX_NAME) as connection: + return (connection.execute("SELECT count(*) FROM attempts").fetchone()[0], + connection.execute("SELECT count(*) FROM records").fetchone()[0]) + + +def replay_only(case, monkeypatch): + def forbidden(*args, **kwargs): + pytest.fail("replay attempted artifact IO, source binding, clock reset or execution") + + class Never: + open = forbidden + + monkeypatch.setattr(execution_service.local_process, "execute_command", forbidden) + monkeypatch.setattr(execution_service.entry_code, "capture_module", forbidden) + monkeypatch.setattr(execution_service, "_deadline_seconds", forbidden) + retired = LocalCpuTarget("retired", "2", work_root=case["target"].work_root / "retired") + return Harness(case["store"], retired, artifact_reader=Never()) + + +def test_completed_evaluation_replays_the_original_ref_without_rebinding(evaluation, monkeypatch): + case = evaluation + saved = deepcopy(case["spec"]) + record = case["harness"].execute(saved) + assert record["started"] is True and record["exit_code"] == 0 + case["modules"].joinpath("fixture_worker.py").unlink() + replayed = replay_only(case, monkeypatch).execute(saved) + assert replayed["record_ref"] == record["record_ref"] + assert replayed["target_binding"] == record["target_binding"] + assert replayed["request_digest"] == run_store.request_digest(saved) + assert replayed["budget_admission"] == record["budget_admission"] + assert saved == case["spec"] + assert counts(case) == (1, 1) + + +@pytest.mark.parametrize("field", ["deadline", "budget", "prepared-ref"]) +def test_changed_evaluation_digest_conflicts_without_any_binding(evaluation, monkeypatch, field): + case = evaluation + original = case["harness"].execute(case["spec"]) + changed = deepcopy(case["spec"]) + if field == "deadline": + changed["deadline_utc"] = "2098-01-01T00:00:00Z" + elif field == "budget": + changed["budget"]["scope_id"] = "another-scope" + else: + changed["prepared_ref"]["artifact"]["sha256"] = "0" * 64 + with pytest.raises(PublicError) as caught: + replay_only(case, monkeypatch).execute(changed) + assert caught.value.code == "RequestConflict" + assert counts(case) == (1, 1) + assert case["store"].read(original["attempt_id"])["record_ref"] == original["record_ref"] + + +@pytest.mark.parametrize("live", [True, False], ids=["live-owner", "owner-gone"]) +def test_existing_attempt_without_a_record_never_implicitly_starts(evaluation, monkeypatch, live): + case = evaluation + attempt = {key: case["spec"][key] for key in execution_service.ATTEMPT_FIELDS} + claim = case["store"].claim(attempt, run_store.request_digest(case["spec"])) + try: + if not live: + claim.release() + with pytest.raises(PublicError) as caught: + replay_only(case, monkeypatch).execute(case["spec"]) + assert caught.value.code == ("AttemptInProgress" if live else "RecoveryRequired") + assert caught.value.latest_record_ref is None + assert counts(case) == (1, 0) + finally: + claim.release() + + +def test_caller_mutation_during_preparation_read_cannot_change_the_saved_request( + evaluation, +): + case = evaluation + original = deepcopy(case["spec"]) + + class Mutating: + def open(self, ref): + case["spec"]["budget"]["scope_id"] = "mutated-caller" + case["spec"]["output_root"] = "mutated-caller" + return case["reader"].open(ref) + + harness = Harness(case["store"], case["target"], artifact_reader=Mutating()) + result = harness.execute(case["spec"]) + assert result["request_digest"] == run_store.request_digest(original) + assert result["budget_admission"]["scope_id"] == original["budget"]["scope_id"] + assert result["started"] is True and result["exit_code"] == 0 From 66fef981f6893038f3128595ffcdfd12b65bfba2 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 21:06:02 +0800 Subject: [PATCH 7/8] test(harness): guard the actual supervisor before registration Intercept the supervisor used by Harness rather than the legacy execution wrapper in zero-start binding tests. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- tests/test_harness_prepared_binding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_harness_prepared_binding.py b/tests/test_harness_prepared_binding.py index fb16b9e9a..61027fac3 100644 --- a/tests/test_harness_prepared_binding.py +++ b/tests/test_harness_prepared_binding.py @@ -62,7 +62,7 @@ def forbidden(*args, **kwargs): pytest.fail("a PR-1 refusal reached registration or process execution") monkeypatch.setattr(store, "claim", forbidden) - monkeypatch.setattr(execution_service.local_process, "execute_command", forbidden) + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) monkeypatch.setattr(execution_service, "_argv", forbidden) return spec, reader, store, harness From 6a2d68f11d576ebfb7dbdb5d27f0fda8928bfb6a Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 21:24:08 +0800 Subject: [PATCH 8/8] feat(harness): preserve original evaluation limits and replay outcomes Stop known expired or cancelled evaluation requests before source capture. Cover real null-timeout expiry, cancellation and descendant cleanup, durable cross-process replay, live owners, commit failure and registration races. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/harness-entry-verification.md | 6 +- docs/contracts/harness-evaluation-replay.md | 61 +++++ docs/contracts/harness-prepared-binding.md | 13 +- docs/operations.md | 1 + src/inferrsi/harness/execution_service.py | 12 +- .../harness-prepared-binding/README.md | 5 +- tests/test_harness_entry_verification.py | 41 ++- tests/test_harness_evaluation_replay.py | 250 +++++++++++++++++- tests/test_harness_prepared_binding.py | 2 +- 9 files changed, 368 insertions(+), 23 deletions(-) create mode 100644 docs/contracts/harness-evaluation-replay.md diff --git a/docs/contracts/harness-entry-verification.md b/docs/contracts/harness-entry-verification.md index 5d3fe1993..e345a1fb0 100644 --- a/docs/contracts/harness-entry-verification.md +++ b/docs/contracts/harness-entry-verification.md @@ -60,6 +60,6 @@ control. CPU execution facts and exit zero are not a Benchmark verdict. The production preparation/Apps archive cross-tests use the accepted D023 / `t108-v4` family and actual source bytes. Their small software worker does not claim complete F/C/M evidence, a service gain, or whole-Issue production -acceptance. Existing ordinary CPU supervision is reused; evaluation deadline, -budget and T40 acceptance remain the later Harness unit, including its separate -launch-boundary prerequisite. +acceptance. Existing ordinary CPU supervision is reused. The +[original-limit/T40 contract](harness-evaluation-replay.md) describes the next +unit; complete CPU acceptance retains its separate launch-boundary prerequisite. diff --git a/docs/contracts/harness-evaluation-replay.md b/docs/contracts/harness-evaluation-replay.md new file mode 100644 index 000000000..ba64bad80 --- /dev/null +++ b/docs/contracts/harness-evaluation-replay.md @@ -0,0 +1,61 @@ +# Harness evaluation replay and original limits + +`Harness.execute` accepts the saved canonical ExecutionSpec. Static schema and +field checks run first. Harness then snapshots the request and checks its +existing RunStore identity/digest before reading preparation bytes, resolving a +target, admitting a budget, binding source or starting a process. + +| Existing state | Outcome | +| --- | --- | +| Finished committed attempt, same digest | Original fixed RecordRef and payload; no new binding, registration or worker | +| Same attempt, different digest | RequestConflict before artifact IO or target checks | +| Live owner, no terminal record | AttemptInProgress; attempt count unchanged | +| Owner gone or persistence failed without a record | RecoveryRequired; no implicit resume or new worker | +| No registered attempt | Validate fixed preparation/support, then atomically claim and execute | + +The non-registering lookup does not replace the atomic claim. A caller can +register while another is reading the preparation; the later claim still +applies the same digest/state table. A corrupt or unavailable RunStore is an +error, never permission to start again. Lookup failures do not fabricate a +known start state. + +Finished positive and refusal records both remain fixed. Repairing a missing +or mismatching source does not turn replay into a new run. The currently +configured target may be retired and the preparation reader unavailable on the +Harness replay path. The original record remains authoritative. Apps performs +its own saved-request binding before handing the ExecutionSpec to Harness; this +does not change Apps archive retention requirements. + +## Identity, deadline and budget + +The original request retains its attempt, request digest, target request, +deadline and complete budget. Host callbacks receive no mutable alias to the +internal request snapshot. A changed deadline, budget scope or preparation Ref +changes the digest and conflicts with an existing attempt. Reentry never +allocates a new identity, resets a deadline, reserves capacity or charges again. + +Fresh evaluation requests keep the original absolute attempt deadline when +step timeout is null. Known expiry/cancellation stops before source capture. +Mid-run expiry and explicit caller cancellation use the existing process-domain +supervisor and retain process exit, cleanup and lease release as separate facts. + +The initial CPU target keeps the existing free-CPU budget support boundary. +Unsupported policies, dimensions or enforcement requirements are explicitly +rejected before worker binding. Reserved/charged amounts remain unknown or +inapplicable rather than invented zeros. Actual observed usage and the original +budget admission remain in the fixed record. This slice adds no budget provider, +currency model or numeric acceptance gate. + +## Verification boundary + +The software tests combine real production preparation, Apps archives and the +configured source target. They cover completed/refused replay, digest conflicts, +live-owner and abandoned state, real running replay, real null-timeout expiry, +caller cancellation with a descendant, fixed cleanup facts, commit failure after +a real start, cross-process replay and registration races. Execution facts do +not provide a complete Benchmark F/C/M proof or a service-gain claim. + +The launch-boundary prerequisite still has its own delivery and review. The +deadline/step-timeout check during log setup must be integrated and verified +before whole CPU acceptance of this slice. The replay implementation does not +duplicate that supervisor change or close the governing Harness/Apps Issues. diff --git a/docs/contracts/harness-prepared-binding.md b/docs/contracts/harness-prepared-binding.md index 0f4d75804..16b39589f 100644 --- a/docs/contracts/harness-prepared-binding.md +++ b/docs/contracts/harness-prepared-binding.md @@ -31,8 +31,10 @@ with the fixed preparation. Disagreement is `IdentityMismatch`. It also checks: and input/output paths must not overlap. Required frozen capabilities cannot be dropped; additional declared capabilities remain subject to admission. -Malformed or conflicting declarations are `InvalidExecutionSpec`. All of these -refusals precede `RunStore.claim`: neither attempt nor record is registered. +Malformed or conflicting declarations are `InvalidExecutionSpec`. For a fresh +request these refusals precede `RunStore.claim`: neither attempt nor record is +registered. Existing request state is handled after static field checks and +before preparation IO by the [replay contract](harness-evaluation-replay.md). The public projection does not interpret candidate formats, grading rules, evidence requirements or scores, and does not recompute Benchmark's semantic prepared ID. The fixed ArtifactRef authenticates the preparation bytes; Apps @@ -47,7 +49,8 @@ The vector is documentary; its historical code pin is not an available worker. Targets without an explicit `module_root` keep valid evaluation-mode execution unsupported before registration. A configured source target performs the separate -[post-registration entry check](harness-entry-verification.md) (T37). Attempt -deadlines/budget acceptance with a null step timeout and evaluation replay (T40) -remain separate delivery work. Declaration checks alone do not establish +[post-registration entry check](harness-entry-verification.md) (T37). The +[original-limit/replay path](harness-evaluation-replay.md) covers the next +delivery unit, whose complete CPU acceptance retains its launch prerequisite. +Declaration checks alone do not establish production CPU acceptance or completion of the governing Harness or Apps Issues. diff --git a/docs/operations.md b/docs/operations.md index 1868dfb2c..ac3ac527b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -54,6 +54,7 @@ Effect classes used below: | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | | Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) | | Verify and execute one frozen source evaluation entry | Python `inferrsi.harness.api.Harness.execute` with trusted `LocalCpuTarget(module_root=...)` | Experiment-executing / record-writing | Fixed preparation and expected code digest; actual source captured after registration; T37 refusals start no worker; [source-entry contract](contracts/harness-entry-verification.md) | +| Replay the same saved execution request | Python `inferrsi.harness.api.Harness.execute` | Existing-state query / fixed record-reading | Same attempt and canonical digest; existing state short-circuits fresh binding/execution; [replay and original limits](contracts/harness-evaluation-replay.md) | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | diff --git a/src/inferrsi/harness/execution_service.py b/src/inferrsi/harness/execution_service.py index 5b5942360..ed755a459 100644 --- a/src/inferrsi/harness/execution_service.py +++ b/src/inferrsi/harness/execution_service.py @@ -946,7 +946,7 @@ def _refusal(ctx, digest, binding, admission, bound, checks, errors, reason) -> ctx, request_digest=digest, target_binding=binding, bound_inputs=bound, phase="finished", started=False, process_outcome="not_started", exit_code=None, steps=[_not_started_step(step, reason) for step in ctx.spec["steps"]], - cancellation={"requested": False, "requested_at": None, "source": None}, + cancellation=_cancellation(ctx, []), cleanup=cleanup, checks=checks, budget_admission=admission, usage=_usage([]), resource_leases=leases, errors=errors, ) @@ -960,6 +960,16 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: if errors: return _refusal(ctx, digest, binding, admission, [], [], errors, "admission_rejected") evaluation = ctx.spec["prepared_ref"] is not None + if evaluation: + if cancellation is not None and cancellation.is_set(): + ctx.cancel_seen_at = run_store.utc_now() + return _refusal(ctx, digest, binding, admission, [], [], [], "cancelled") + if time.monotonic() >= deadline: + return _refusal( + ctx, digest, binding, admission, [], [], + [_error("deadline_exhausted", "binding", "original attempt deadline exhausted")], + "deadline_exhausted", + ) entry_checks = [] if evaluation: entries = [_bind_entry(ctx, step, target) for step in ctx.spec["steps"]] diff --git a/tests/fixtures/harness-prepared-binding/README.md b/tests/fixtures/harness-prepared-binding/README.md index 5d6e17898..8d0d8b008 100644 --- a/tests/fixtures/harness-prepared-binding/README.md +++ b/tests/fixtures/harness-prepared-binding/README.md @@ -11,6 +11,7 @@ This is immutable contract-conformance input, not observed execution evidence or an available worker implementation. Tests compare the canonical Apps binder, Harness digest and read-only prepared projection. No process is started by these binding tests. Post-registration T37 is covered separately in -`tests/test_harness_entry_verification.py`; deadline/budget and production T40 -replay remain PR-3 work. +`tests/test_harness_entry_verification.py`; deadline/budget and T40 replay have +their own tests in `tests/test_harness_evaluation_replay.py`. Whole production +integration retains its separate acceptance dependencies. The original documentary fixtures are not rewritten. diff --git a/tests/test_harness_entry_verification.py b/tests/test_harness_entry_verification.py index aa42b0195..727c213f4 100644 --- a/tests/test_harness_entry_verification.py +++ b/tests/test_harness_entry_verification.py @@ -5,6 +5,7 @@ import subprocess import sys from copy import deepcopy +from datetime import datetime, timedelta, timezone from hashlib import sha256 from pathlib import Path @@ -26,10 +27,22 @@ @pytest.fixture -def evaluation(preparation, tmp_path): - service, request, sources, _, _, config, _ = preparation +def evaluation(preparation, tmp_path, request): + service, prepare_request, sources, protocol, _, config, _ = preparation + settings = getattr(request, "param", {}) + if "worker_code" in settings: + source = settings["worker_code"] + sources["worker-code"].write_bytes(source) + protocol = deepcopy(protocol) + digest = sha256(source).hexdigest() + protocol["worker_recipe"]["code_sha256"] = digest + resource = next(item for item in protocol["resources"] if item["role"] == "worker-code") + resource["artifact"].update(sha256=digest, size_bytes=len(source)) + service = benchmark.PreparationService( + protocol, resources=lambda role: sources[role].open("rb"), worker_role="worker-code", + ) output = api.prepare( - {"schema_version": 1, "request": request}, workspace=tmp_path, + {"schema_version": 1, "request": prepare_request}, workspace=tmp_path, storage_config=config, prepare_benchmark=service.prepare, encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, ) @@ -38,6 +51,10 @@ def evaluation(preparation, tmp_path): prepared = read_prepared(config, ref) vector = decode((ROOT / "tests/fixtures/harness-prepared-binding/d023-null-timeout.json") .read_bytes()) + if "deadline_seconds" in settings: + vector["deadline_utc"] = ( + datetime.now(timezone.utc) + timedelta(seconds=settings["deadline_seconds"]) + ).isoformat().replace("+00:00", "Z") controller, modules = tmp_path / "controller", tmp_path / "modules" controller.mkdir() modules.mkdir() @@ -108,7 +125,7 @@ def test_t37_entry_failure_commits_a_not_started_record(evaluation, monkeypatch, def forbidden(*args, **kwargs): pytest.fail("T37-N1/N2 launched a process") - monkeypatch.setattr(execution_service.local_process, "execute_command", forbidden) + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) record = case["harness"].execute(case["spec"]) assert_fixed_refusal( case, record, "failed" if failure == "mismatch" else "not_performed", actual, @@ -128,6 +145,15 @@ def test_matching_entry_executes_its_snapshot_and_retains_the_observed_digest( evaluation, monkeypatch, interference, ): case = evaluation + supervise = execution_service.local_process._supervise + observed_pids = [] + + def counted(*args, **kwargs): + result = supervise(*args, **kwargs) + observed_pids.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) if interference == "workspace-shadow": case["spec"]["inputs"][0]["materialized_path"] = "fixture_worker.py" case["spec"]["steps"][0]["args"][0] = "fixture_worker.py" @@ -145,6 +171,7 @@ def capture_then_change(*args, **kwargs): record = case["harness"].execute(case["spec"]) assert record["started"] is True and record["process_outcome"] == "exited" assert record["exit_code"] == 0 + assert len(observed_pids) == 1 and observed_pids[0] is not None check = record["checks"][0] assert check["status"] == "passed" actual = sha256(case["source"]).hexdigest() @@ -170,7 +197,7 @@ def fail_commit(*args, **kwargs): raise sqlite3.OperationalError("fixture commit failure") monkeypatch.setattr(case["store"], "_append", fail_commit) - monkeypatch.setattr(execution_service.local_process, "execute_command", + monkeypatch.setattr(execution_service.local_process, "_supervise", lambda *a, **k: pytest.fail("failed T37 commit started a process")) with pytest.raises(PublicError) as caught: case["harness"].execute(case["spec"]) @@ -191,7 +218,7 @@ def test_t37_fixed_mismatch_and_source_bytes_can_be_read_in_a_fresh_process( changed = b"raise RuntimeError('mismatching worker never executes')\n" source = case["modules"] / "fixture_worker.py" source.write_bytes(changed) - monkeypatch.setattr(execution_service.local_process, "execute_command", + monkeypatch.setattr(execution_service.local_process, "_supervise", lambda *a, **k: pytest.fail("mismatching worker started")) record = case["harness"].execute(case["spec"]) source.unlink() @@ -219,7 +246,7 @@ def fail_capture(*args, **kwargs): raise execution_service.entry_code.EntryCaptureError("fixture capture failure") monkeypatch.setattr(execution_service.entry_code, "capture_module", fail_capture) - monkeypatch.setattr(execution_service.local_process, "execute_command", + monkeypatch.setattr(execution_service.local_process, "_supervise", lambda *a, **k: pytest.fail("failed capture started a worker")) with pytest.raises(PublicError) as caught: case["harness"].execute(case["spec"]) diff --git a/tests/test_harness_evaluation_replay.py b/tests/test_harness_evaluation_replay.py index aae8a5616..570b50087 100644 --- a/tests/test_harness_evaluation_replay.py +++ b/tests/test_harness_evaluation_replay.py @@ -1,10 +1,17 @@ """Evaluation reentry is decided before preparation, target binding or a new launch.""" +import json import sqlite3 +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor from copy import deepcopy import pytest +from inferrsi.contracts.json_v1 import encode from inferrsi.harness import execution_service, run_store from inferrsi.harness.api import Harness, LocalCpuTarget from inferrsi.harness.errors import PublicError @@ -25,19 +32,34 @@ def forbidden(*args, **kwargs): class Never: open = forbidden - monkeypatch.setattr(execution_service.local_process, "execute_command", forbidden) + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) monkeypatch.setattr(execution_service.entry_code, "capture_module", forbidden) monkeypatch.setattr(execution_service, "_deadline_seconds", forbidden) + monkeypatch.setattr(execution_service, "_budget", forbidden) + monkeypatch.setattr(execution_service, "_capabilities", forbidden) retired = LocalCpuTarget("retired", "2", work_root=case["target"].work_root / "retired") return Harness(case["store"], retired, artifact_reader=Never()) -def test_completed_evaluation_replays_the_original_ref_without_rebinding(evaluation, monkeypatch): +@pytest.mark.parametrize("initial", ["exited", "mismatch", "missing"]) +def test_completed_evaluation_replays_the_original_ref_without_rebinding( + evaluation, monkeypatch, initial, +): case = evaluation saved = deepcopy(case["spec"]) + source = case["modules"] / "fixture_worker.py" + if initial == "mismatch": + source.write_bytes(b"changed source\n") + elif initial == "missing": + source.unlink() record = case["harness"].execute(saved) - assert record["started"] is True and record["exit_code"] == 0 - case["modules"].joinpath("fixture_worker.py").unlink() + assert record["started"] is (initial == "exited") + if initial == "exited": + assert record["exit_code"] == 0 + source.unlink() + else: + # Repairing the source does not authorize a new execution of this attempt. + source.write_bytes(case["source"]) replayed = replay_only(case, monkeypatch).execute(saved) assert replayed["record_ref"] == record["record_ref"] assert replayed["target_binding"] == record["target_binding"] @@ -65,6 +87,31 @@ def test_changed_evaluation_digest_conflicts_without_any_binding(evaluation, mon assert case["store"].read(original["attempt_id"])["record_ref"] == original["record_ref"] +def test_failed_commit_after_a_real_start_never_replays_the_worker(evaluation, monkeypatch): + case = evaluation + supervise = execution_service.local_process._supervise + observed = [] + + def counted(*args, **kwargs): + result = supervise(*args, **kwargs) + observed.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + monkeypatch.setattr(case["store"], "_append", + lambda *a, **k: (_ for _ in ()).throw(sqlite3.OperationalError("fixture"))) + with pytest.raises(PublicError) as failed: + case["harness"].execute(case["spec"]) + assert failed.value.code == "ExecutionPersistenceError" and failed.value.started is True + assert len(observed) == 1 and observed[0] is not None + assert counts(case) == (1, 0) + with pytest.raises(PublicError) as incomplete: + replay_only(case, monkeypatch).execute(case["spec"]) + assert incomplete.value.code == "RecoveryRequired" + assert incomplete.value.started is not False + assert counts(case) == (1, 0) and len(observed) == 1 + + @pytest.mark.parametrize("live", [True, False], ids=["live-owner", "owner-gone"]) def test_existing_attempt_without_a_record_never_implicitly_starts(evaluation, monkeypatch, live): case = evaluation @@ -99,3 +146,198 @@ def open(self, ref): assert result["request_digest"] == run_store.request_digest(original) assert result["budget_admission"]["scope_id"] == original["budget"]["scope_id"] assert result["started"] is True and result["exit_code"] == 0 + + +SLEEPING_WORKER = ( + b"import json,pathlib,subprocess,sys,time\n" + b"child=subprocess.Popen([sys.executable,'-I','-B','-c','import time; time.sleep(60)'])\n" + b"out=pathlib.Path(sys.argv[2]); out.parent.mkdir(parents=True,exist_ok=True)\n" + b"out.write_text(json.dumps({'child_pid':child.pid}),encoding='utf-8')\n" + b"time.sleep(60)\n" +) + + +def assert_confirmed_cleanup(record): + assert record["cleanup"]["status"] == "confirmed" + assert record["cleanup"]["process_exit_confirmed"] is True + assert record["resource_leases"]["leases"] + assert all(lease["release_confirmed"] is True + for lease in record["resource_leases"]["leases"]) + assert all(lease["release_confirmed_at"] is not None + for lease in record["resource_leases"]["leases"]) + + +@pytest.mark.parametrize("evaluation", [ + {"worker_code": SLEEPING_WORKER, "deadline_seconds": 3}, +], indirect=True) +def test_null_step_timeout_uses_original_attempt_deadline_and_cleans_the_domain(evaluation): + case = evaluation + saved = deepcopy(case["spec"]) + assert saved["steps"][0]["timeout_seconds"] is None + record = case["harness"].execute(saved) + assert record["started"] is True and record["process_outcome"] == "timed_out" + assert record["steps"][0]["status"] == "timed_out" + assert record["request_digest"] == run_store.request_digest(saved) + assert saved["deadline_utc"] == case["spec"]["deadline_utc"] + assert record["budget_admission"]["scope_id"] == saved["budget"]["scope_id"] + assert_confirmed_cleanup(record) + + +@pytest.mark.parametrize("evaluation", [ + {"worker_code": SLEEPING_WORKER, "deadline_seconds": 20}, +], indirect=True) +def test_live_cpu_replay_and_cancellation_preserve_identity_without_new_starts( + evaluation, monkeypatch, +): + case = evaluation + saved = deepcopy(case["spec"]) + cancellation = threading.Event() + command = execution_service.local_process._supervise + starts = [] + + def counted(*args, **kwargs): + starts.append(1) + return command(*args, **kwargs) + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + marker = case["target"].work_root / "workspaces" / saved["output_root"] / "outputs/grader.json" + with ThreadPoolExecutor(max_workers=1) as pool: + original = pool.submit(case["harness"].execute, saved, cancellation=cancellation) + try: + stop = time.monotonic() + 10 + while not marker.exists() and not original.done() and time.monotonic() < stop: + time.sleep(0.01) + assert marker.exists(), "the real worker did not publish its running marker" + assert starts == [1] + assert counts(case) == (1, 0) + replay = replay_only(case, monkeypatch) + with pytest.raises(PublicError) as caught: + replay.execute(saved) + assert caught.value.code == "AttemptInProgress" + assert counts(case) == (1, 0) and starts == [1] + changed = deepcopy(saved) + changed["budget"]["scope_id"] = "conflict-during-run" + with pytest.raises(PublicError) as conflict: + replay.execute(changed) + assert conflict.value.code == "RequestConflict" + assert starts == [1] + finally: + cancellation.set() + record = original.result(timeout=15) + assert record["started"] is True and record["process_outcome"] == "cancelled" + assert record["cancellation"]["requested"] is True + assert record["cancellation"]["source"] == "caller" + assert_confirmed_cleanup(record) + again = replay.execute(saved) + assert again["record_ref"] == record["record_ref"] + assert counts(case) == (1, 1) and starts == [1] + + +@pytest.mark.parametrize("limit", ["expired", "cancelled", "budget"]) +def test_known_limits_refuse_before_source_capture_and_preserve_original_budget( + evaluation, monkeypatch, limit, +): + case = evaluation + saved = deepcopy(case["spec"]) + cancellation = threading.Event() + if limit == "expired": + saved["deadline_utc"] = "2000-01-01T00:00:00Z" + elif limit == "cancelled": + cancellation.set() + else: + saved["budget"]["dimensions"] = [ + {"dimension": "wall-time", "unit": "second", "limit": "1", "reserve_estimate": None}, + ] + + def forbidden(*args, **kwargs): + pytest.fail("a refused original limit reached source capture or process execution") + + monkeypatch.setattr(execution_service.entry_code, "capture_module", forbidden) + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + record = case["harness"].execute(saved, cancellation=cancellation) + assert record["started"] is False and record["process_outcome"] == "not_started" + assert record["request_digest"] == run_store.request_digest(saved) + assert record["budget_admission"]["scope_id"] == saved["budget"]["scope_id"] + if limit == "cancelled": + assert record["cancellation"]["requested"] is True + elif limit == "budget": + assert record["budget_admission"]["decision"] == "rejected" + assert all(item["reserved_capacity"] is None and item["charged_capacity"] is None + for item in record["budget_admission"]["dimensions"]) + assert counts(case) == (1, 1) + + +def test_completed_request_replays_in_a_new_controller_without_archive_or_target(evaluation): + case = evaluation + saved = deepcopy(case["spec"]) + record = case["harness"].execute(saved) + case["modules"].joinpath("fixture_worker.py").unlink() + script = ( + "import json,sys; from pathlib import Path; " + "from inferrsi.contracts.json_v1 import decode; " + "from inferrsi.harness import execution_service; " + "from inferrsi.harness.api import Harness,LocalCpuTarget; " + "from inferrsi.harness.run_store import RunStore; " + "s=RunStore(sys.argv[1],'test'); spec=decode(sys.stdin.buffer.read()); " + "target=LocalCpuTarget('retired','2',work_root=Path(sys.argv[1]).parent/'retired'); " + "execution_service.prepared_binding.verify=lambda *a,**k: " + "(_ for _ in ()).throw(RuntimeError('replay read preparation')); " + "execution_service.local_process._supervise=lambda *a,**k: " + "(_ for _ in ()).throw(RuntimeError('replay started a worker')); " + "r=Harness(s,target).execute(spec); " + "print(json.dumps([r['record_ref'],r['request_digest'],r['budget_admission']]))" + ) + child = subprocess.run( + [sys.executable, "-c", script, str(case["store"].root)], input=encode(saved), + check=True, capture_output=True, timeout=30, + ) + assert json.loads(child.stdout) == [ + record["record_ref"], record["request_digest"], record["budget_admission"], + ] + assert counts(case) == (1, 1) + + +def test_registration_race_after_lookup_still_uses_the_atomic_claim(evaluation, monkeypatch): + case = evaluation + saved = deepcopy(case["spec"]) + attempt = {key: saved[key] for key in execution_service.ATTEMPT_FIELDS} + held = [] + + class Racing: + def open(self, ref): + if not held: + held.append(case["store"].claim(attempt, run_store.request_digest(saved))) + return case["reader"].open(ref) + + monkeypatch.setattr(execution_service.local_process, "_supervise", + lambda *a, **k: pytest.fail("registration race started another worker")) + monkeypatch.setattr(execution_service.entry_code, "capture_module", + lambda *a, **k: pytest.fail("registration race bound a new entry")) + try: + with pytest.raises(PublicError) as caught: + Harness(case["store"], case["target"], artifact_reader=Racing()).execute(saved) + assert caught.value.code == "AttemptInProgress" + assert len(held) == 1 and counts(case) == (1, 0) + finally: + for claim in held: + claim.release() + + +@pytest.mark.parametrize("malformed", ["version", "missing-code"]) +def test_static_shape_errors_still_precede_existing_state_lookup( + evaluation, monkeypatch, malformed, +): + case = evaluation + original = case["harness"].execute(case["spec"]) + changed = deepcopy(case["spec"]) + if malformed == "version": + changed["schema_version"] = 2 + else: + del changed["steps"][0]["expected_code_sha256"] + monkeypatch.setattr(case["store"], "_reenter_if_registered", + lambda *a, **k: pytest.fail("malformed request queried existing state")) + with pytest.raises(PublicError) as caught: + case["harness"].execute(changed) + assert caught.value.code == ("UnsupportedSchema" if malformed == "version" + else "InvalidExecutionSpec") + assert case["store"].read(original["attempt_id"])["record_ref"] == original["record_ref"] diff --git a/tests/test_harness_prepared_binding.py b/tests/test_harness_prepared_binding.py index fb16b9e9a..61027fac3 100644 --- a/tests/test_harness_prepared_binding.py +++ b/tests/test_harness_prepared_binding.py @@ -62,7 +62,7 @@ def forbidden(*args, **kwargs): pytest.fail("a PR-1 refusal reached registration or process execution") monkeypatch.setattr(store, "claim", forbidden) - monkeypatch.setattr(execution_service.local_process, "execute_command", forbidden) + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) monkeypatch.setattr(execution_service, "_argv", forbidden) return spec, reader, store, harness