From ab61aebe745a6110902d15aa70d1129f9337d3d7 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 18:58:59 +0800 Subject: [PATCH 1/5] feat(benchmark): add frozen P1 offline assessment Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/apps-module-migrations.json | 9 +- docs/contracts/production-assess.md | 89 +++++ docs/operations.md | 1 + src/inferrsi/benchmark/assessment.py | 551 +++++++++++++++++++++++++++ tests/test_benchmark_assessment.py | 539 ++++++++++++++++++++++++++ 5 files changed, 1186 insertions(+), 3 deletions(-) create mode 100644 docs/contracts/production-assess.md create mode 100644 src/inferrsi/benchmark/assessment.py create mode 100644 tests/test_benchmark_assessment.py diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index ebaf1b65..99852a52 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -37,7 +37,8 @@ "tests/test_apps_publication.py", "tests/test_apps_replay.py", "tests/test_benchmark_preparation.py", - "tests/test_result_delivery.py" + "tests/test_result_delivery.py", + "tests/test_benchmark_assessment.py" ] }, { @@ -60,7 +61,8 @@ "tests/test_apps_artifact_reader.py", "tests/test_apps_assess.py", "tests/test_apps_generation_plan.py", - "tests/test_apps_module_migrations.py" + "tests/test_apps_module_migrations.py", + "tests/test_benchmark_assessment.py" ] }, { @@ -139,7 +141,8 @@ "tests/test_apps_execution_binding.py", "tests/test_apps_generation_plan.py", "tests/test_apps_module_migrations.py", - "tests/test_benchmark_preparation.py" + "tests/test_benchmark_preparation.py", + "tests/test_benchmark_assessment.py" ] }, { diff --git a/docs/contracts/production-assess.md b/docs/contracts/production-assess.md new file mode 100644 index 00000000..834f8bbc --- /dev/null +++ b/docs/contracts/production-assess.md @@ -0,0 +1,89 @@ +# Production P1 offline assessment + +`inferrsi.benchmark.assessment` implements the accepted D023 / `t108-v4` +family: `p1-fixture@1`, `p1-rules@1`, operation `evaluate`, scope `p1`. +It consumes the same frozen preparation as +[production prepare](production-prepare.md). The public ports are +`assess(prepared, evidence, *, artifacts)`, `encode`, `decode`, and +`validate_assessment`. `AssessmentService(clock=...)` supports reproducible +assessment timestamps; its default clock is real UTC. + +```python +from inferrsi.apps.api import assess as apps_assess +from inferrsi.benchmark.assessment import assess, encode, validate_assessment + +output = apps_assess( + saved_assess_request, workspace=workspace, storage_config=storage_config, + assess_benchmark=assess, encode_assessment=encode, + validate_assessment=validate_assessment, +) +``` + +Apps reads one fixed record and its preparation, checks the primary key against +the frozen protocol, and supplies its read-only artifact capability. Benchmark +does not import Apps, Harness or Knowledge, register a worker, execute a process, +use a network/device, archive results, or rewrite execution facts. Its encoder, +decoder and result validator perform no artifact reads. + +## Binding and byte authority + +The port snapshots the JSON inputs and checks the complete canonical record +against its RecordRef hash, size, attempt and revision. It checks task, +operation, scope, semantic `prepared_id` and `protocol_digest`. It does not +compare archived reference locations as identities. The actual frozen protocol +bytes are read and validated with the prepare port's pure validator; unsupported +task, grader or claim-rule revisions are explicitly refused. + +Every opened artifact is independently checked against its declared size and +SHA-256. The capability may supply bytes or a binary stream; assessment closes +streams it opens. Portable `UnsafeArtifactPath`, `ArtifactUnavailable` and +`ArtifactIntegrityError` exceptions are recognized by their exact `code`, without +depending on another module's exception class. An affected proof becomes a +claim-level evidence gap. Unknown host programming exceptions propagate. +Failure to obtain the protocol itself is a port error: missing rules cannot be +replaced by a guessed assessment. + +An input's actual hash is accepted only with matching retained bytes and a fixed +capture fact. Proof-declared candidate IDs/hashes do not authenticate themselves. +Check identity, revision, subject, phase, attempt, operation, scope and artifact +schema must match a frozen requirement. Producer kind/domain must be allowed; +the domain must match the actual target binding. Worker and sample evidence must +also match the observed runtime entry/revision/code pin and a started step bound +to that input. Caller assertions cannot substitute for those facts. + +## Frozen P1 decisions + +- F and C consume their own trustworthy format/grader proofs. A complete passed + proof yields PASS, a matching failed proof yields FAIL, and missing/invalid + evidence yields INCONCLUSIVE. Another claim's missing evidence does not erase + an independently supported result. +- M validates four positive Decimal samples in A1/B1/B2/A2 order with their + frozen round, role and `ms` unit. It also needs a completed successful process, + known closed steps, confirmed process cleanup, confirmed acquired-lease + releases and a matching cleanup proof. Missing terminal facts remain + INCONCLUSIVE; there is no independent-error exception for this terminal claim. +- Contradictory trustworthy proofs are aggregated into one INCONCLUSIVE claim + with a conflict reason, including conflicting sample values. No first/last + winner or duplicate claim key is emitted. P1 eligibility is ineligible for a + failed claim/conflict, unknown for incomplete evidence, and eligible only when + all three frozen conditions pass. It is not an overall verdict. +- This protocol freezes no metric definitions or assessment comparison target, + so metrics/comparison arrays remain empty. M is measurement validity, not a + claim of a service gain. + +Raw proof bytes retain their original hash. Their JSON need not use the DTO's +canonical whitespace: parsing still rejects duplicate keys, floating/nonfinite +numeric tokens, invalid Unicode and invalid P1 fields. New EvaluationResult +bytes follow `inferrsi-json-v1`. Its semantic ID excludes only the declared +clock/self-ID fields and schema-typed ArtifactRef locations, including nested +candidate references. All other supporting content participates. + +## Acceptance boundary + +The Apps primary-key order, missing-result-key error and archive-failure behavior +remain Apps responsibilities. Cross-tests use real production prepare/assess +ports, Apps archive/read/publish code and retained software-test record/proof +fixtures. They cover T35/T39/V21–V23, including isolated-process identity and +fixed-clock bytes. They do not claim that the production Harness evaluation +worker has run. That target and T37/T40 belong to #429; joint production CPU +acceptance remains necessary before #371 can be completed. diff --git a/docs/operations.md b/docs/operations.md index 9115ca36..d5c18a15 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -53,6 +53,7 @@ Effect classes used below: | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | +| Assess one fixed P1 record offline | Python `benchmark.assessment.assess`; [production assess](contracts/production-assess.md) | Read-only retained evidence / pure encoding | Verifies the frozen protocol, actual capture and proof bytes; returns independent claims without launching a worker or writing an archive; Apps selects the primary key and archives the result | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | | Retain an executed retrieval | Python `apps.workflows.campaign.save_retrieval`; [capture contract](knowledge-retrieval-trace.md#retain-a-standalone-result) | Local evidence-writing | Saves an existing catalog/collect result under a fresh campaign capture ID; no new search or automatic publication | diff --git a/src/inferrsi/benchmark/assessment.py b/src/inferrsi/benchmark/assessment.py new file mode 100644 index 00000000..7989dacd --- /dev/null +++ b/src/inferrsi/benchmark/assessment.py @@ -0,0 +1,551 @@ +"""Read-only D023/P1 assessment of one fixed record and its retained evidence.""" + +from __future__ import annotations + +import json +import re +from contextlib import closing +from copy import deepcopy +from datetime import datetime, timezone +from decimal import Decimal +from hashlib import sha256 + +from ..contracts.json_v1 import CanonicalError +from ..contracts.json_v1 import decode as canonical_decode +from ..contracts.json_v1 import encode as canonical_encode +from . import preparation + +_RESULT = set("schema_version assessment_id task grader assessor prepared_ref record_ref " + "protocol_digest candidate attempt_id claims metrics eligibility comparison " + "evidence_artifacts assessed_at supersedes derived_from limitations".split()) +_CLAIM = set("claim_id validation_scope verdict reasons evidence " + "missing_evidence limitations".split()) +_RECORD = set("schema_version request_id run_id attempt_id request_digest owner target_request " + "target_binding retry_of retry_reason generation_record_ref prepared_ref " + "protocol_digest operation validation_scope candidate record_version recorded_at " + "bound_inputs phase started side_effect_unknown process_outcome exit_code " + "elapsed_seconds steps cancellation cleanup artifacts checks budget_admission " + "usage resource_leases errors".split()) +_COMMON = {"schema_version", "candidate_id", "actual_sha256", "attempt_id", "operation", + "validation_scope"} +_ITEM_ERRORS = {"UnsafeArtifactPath", "ArtifactUnavailable", "ArtifactIntegrityError"} + + +class AssessmentError(ValueError): + """A declared Benchmark rejection, independent of any Harness exception class.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + def to_json(self) -> dict: + return { + "code": self.code, + "stage": "assessment_binding" if self.code == "IdentityMismatch" else "assess", + "attempt_id": None, "started": None, "latest_record_ref": None, + "partial_record": None, "retained_refs": [], "field_errors": [], + "message": str(self), + } + + +def _require(condition, message="Invalid P1 assessment.", code="InvalidRequest"): + if not condition: + raise AssessmentError(code, message) + + +def _object(value, fields): + _require(isinstance(value, dict) and value.keys() == fields) + + +def _artifact(value): + preparation._artifact(value) + _require(all(ord(char) >= 32 and ord(char) != 127 for char in value["location"]), + "Artifact paths must not contain control characters.") + + +def _strings(value, *, identifiers=False): + _require(isinstance(value, list) and all( + preparation._id(item) if identifiers else isinstance(item, str) for item in value + )) + + +def _timestamp(value): + _require(isinstance(value, str) and re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z", value + ) is not None) + try: + datetime.fromisoformat(value[:19]) + except ValueError: + raise AssessmentError("InvalidRequest", "Invalid assessment timestamp.") from None + + +def _projection(result): + """Project only schema-declared ArtifactRefs, including nested candidate refs.""" + # JSON values have no alias identity: callers may reuse the same dict at + # several typed paths without making the projection depend on object aliasing. + value = canonical_decode(canonical_encode(result)) + del value["assessment_id"], value["assessed_at"] + refs = [value["prepared_ref"], value["record_ref"], *value["derived_from"]] + refs.extend(item["against"] for item in value["comparison"]) + if value["supersedes"] is not None: + refs.append(value["supersedes"]) + artifacts = [ref["artifact"] for ref in refs] + artifacts.extend(value["evidence_artifacts"]) + for items, field in ((value["claims"], "evidence"), (value["metrics"], "raw_samples")): + artifacts.extend(ref for item in items for ref in item[field]) + if value["candidate"] is not None: + artifacts.extend(value["candidate"]["context_refs"]) + for ref in artifacts: + _artifact(ref) + del ref["location"] + return value + + +def assessment_id(result: dict) -> str: + """Pure v1 identity; only assessed_at, the self ID and typed locations are excluded.""" + _object(result, _RESULT) + return "sha256:" + sha256(canonical_encode(_projection(result))).hexdigest() + + +def validate_assessment(result: dict) -> None: + """Pure validation of the supported P1 result and its semantic identity.""" + canonical_encode(result) + _object(result, _RESULT) + _require(type(result["schema_version"]) is int and result["schema_version"] == 1, + code="UnsupportedSchema") + _require(result["task"] == {"name": "p1-fixture", "revision": "1"} + and result["grader"] == {"name": "p1-rules", "revision": "1"} + and result["assessor"] == {"name": "fixture-assessor", "revision": "1"}, + code="UnsupportedRuleVersion") + _timestamp(result["assessed_at"]) + _require(preparation._id(result["attempt_id"])) + _require(isinstance(result["protocol_digest"], str) + and re.fullmatch(r"sha256:[a-f0-9]{64}", result["protocol_digest"]) is not None) + for field, fields in (("prepared_ref", {"schema_version", "prepared_id", "artifact"}), + ("record_ref", {"attempt_id", "record_version", "artifact"})): + _object(result[field], fields) + _artifact(result[field]["artifact"]) + _require(result["prepared_ref"]["schema_version"] == 1 + and type(result["prepared_ref"]["schema_version"]) is int + and re.fullmatch(r"sha256:[a-f0-9]{64}", + result["prepared_ref"]["prepared_id"]) is not None) + _require(result["record_ref"]["attempt_id"] == result["attempt_id"] + and type(result["record_ref"]["record_version"]) is int + and result["record_ref"]["record_version"] > 0) + candidate = result["candidate"] + if candidate is not None: + _object(candidate, {"candidate_id", "kind", "input_role", "parent_id", + "parent_digest", "context_refs"}) + _require(all(preparation._id(candidate[key]) + for key in ("candidate_id", "kind", "input_role"))) + _require(isinstance(candidate["context_refs"], list)) + for ref in candidate["context_refs"]: + _artifact(ref) + _require(candidate["parent_id"] is None or preparation._id(candidate["parent_id"])) + _require(candidate["parent_digest"] is None or isinstance(candidate["parent_digest"], str) + and preparation._SHA.fullmatch(candidate["parent_digest"]) is not None) + _require(isinstance(result["claims"], list) and len(result["claims"]) == 3) + keys = [] + for claim in result["claims"]: + _object(claim, _CLAIM) + keys.append((claim["claim_id"], claim["validation_scope"])) + _require(claim["verdict"] in {"PASS", "FAIL", "INCONCLUSIVE"}) + for field in ("reasons", "missing_evidence"): + _strings(claim[field], identifiers=True) + _strings(claim["limitations"]) + _require(isinstance(claim["evidence"], list)) + for ref in claim["evidence"]: + _artifact(ref) + _require(len(set(keys)) == len(keys) and set(keys) == {("F", "p1"), ("C", "p1"), ("M", "p1")}) + # P1 freezes no metric definitions or assessment comparison target. + _require(result["metrics"] == [] and result["comparison"] == [] + and result["supersedes"] is None and result["derived_from"] == []) + _require(isinstance(result["eligibility"], list) and len(result["eligibility"]) == 1) + eligibility = result["eligibility"][0] + _object(eligibility, {"validation_scope", "status", "reasons"}) + _require(eligibility["validation_scope"] == "p1" + and eligibility["status"] in {"eligible", "ineligible", "unknown"}) + _strings(eligibility["reasons"], identifiers=True) + _strings(result["limitations"]) + _require(isinstance(result["evidence_artifacts"], list)) + for ref in result["evidence_artifacts"]: + _artifact(ref) + _require(result["assessment_id"] == assessment_id(result), "Invalid assessment identity.", + "IdentityMismatch") + + +def encode(result: dict) -> bytes: + validate_assessment(result) + return canonical_encode(result) + + +def decode(data: bytes) -> dict: + result = canonical_decode(data) + _require(canonical_encode(result) == data, "Assessment bytes are not canonical.") + validate_assessment(result) + return result + + +class _RetainedBytes: + def __init__(self, artifacts): + self.artifacts = artifacts + self.cache = {} + self.verified = [] + + def read(self, ref): + try: + _artifact(ref) + except (preparation.PreparationError, AssessmentError): + return None, "invalid_artifact_ref" + key = canonical_encode(ref) + if key in self.cache: + return self.cache[key] + try: + source = self.artifacts.open(deepcopy(ref)) + if isinstance(source, bytes): + data = source + else: + chunks, remaining = [], ref["size_bytes"] + 1 + with closing(source): + while remaining > 0: + chunk = source.read(min(remaining, 1024 * 1024)) + if not isinstance(chunk, bytes): + raise TypeError("ArtifactReader must supply binary bytes or streams.") + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + except Exception as error: + # Portable error codes are the capability contract; importing an Apps + # or Harness exception class would invert the module dependency. + if getattr(error, "code", None) not in _ITEM_ERRORS: + raise + self.cache[key] = (None, error.code) + return self.cache[key] + if len(data) != ref["size_bytes"] or sha256(data).hexdigest() != ref["sha256"]: + self.cache[key] = (None, "ArtifactIntegrityError") + else: + self.cache[key] = (data, None) + self.verified.append(deepcopy(ref)) + return self.cache[key] + + +def _binding(prepared, evidence): + preparation.encode(prepared) + _object(evidence, {"schema_version", "record_ref", "record_digest", "task", "task_source", + "record", "artifact_checks"}) + record, ref = evidence["record"], evidence["record_ref"] + _object(record, _RECORD) + _object(ref, {"attempt_id", "record_version", "artifact"}) + _object(record["prepared_ref"], {"schema_version", "prepared_id", "artifact"}) + _artifact(ref["artifact"]) + _require(type(evidence["schema_version"]) is int and evidence["schema_version"] == 1 + and type(record["schema_version"]) is int and record["schema_version"] == 1) + data = canonical_encode(record) + _require(evidence["task"] == prepared["task"] and evidence["task_source"] == "prepared_ref" + and evidence["record_digest"] == ref["artifact"]["sha256"] + and (sha256(data).hexdigest(), len(data)) + == (ref["artifact"]["sha256"], ref["artifact"]["size_bytes"]) + and record["attempt_id"] == ref["attempt_id"] + and record["record_version"] == ref["record_version"] + and record["prepared_ref"]["prepared_id"] == prepared["prepared_id"] + and all(record[key] == prepared[key] + for key in ("protocol_digest", "operation", "validation_scope")), + "Prepared and fixed record identities differ.", "IdentityMismatch") + for field in ("checks", "bound_inputs", "steps", "artifact_checks"): + _require(isinstance((evidence if field == "artifact_checks" else record)[field], list)) + if record["candidate"] is not None: + _require(isinstance(record["candidate"], dict) + and all(record["candidate"].get(key) == prepared["candidate_contract"][key] + for key in ("kind", "input_role")), + "Candidate contract differs from the frozen preparation.", "IdentityMismatch") + if prepared["expected_candidate"] is not None: + _require(record["candidate"] is not None + and record["candidate"]["candidate_id"] + == prepared["expected_candidate"]["candidate_id"], + "Declared candidate identity differs.", "IdentityMismatch") + return record + + +def _captured_candidate(prepared, record, reader): + candidate = record["candidate"] + if not isinstance(candidate, dict): + return None + matches = [item for item in record["bound_inputs"] if isinstance(item, dict) + and item.get("role") == candidate.get("input_role") + and item.get("attempt_id") == record["attempt_id"]] + if len(matches) != 1 or candidate.get("kind") != prepared["candidate_contract"]["kind"]: + return None + item = matches[0] + ref, digest = item.get("retained_ref"), item.get("actual_sha256") + if not isinstance(ref, dict) or digest != ref.get("sha256") or item.get("captured_at") is None: + return None + try: + _timestamp(item["captured_at"]) + except AssessmentError: + return None + data, _ = reader.read(ref) + return digest if data is not None else None + + +def _trusted(check, required, prepared, record): + fields = {"check_id": required["check_id"], "checker_revision": required["checker_revision"], + "subject": required["subject"], "phase": required["phase"], + "artifact_schema": required["artifact_schema"], "operation": record["operation"], + "validation_scope": record["validation_scope"], "attempt_id": record["attempt_id"]} + if any(check.get(key) != value for key, value in fields.items()): + return False + producer, binding = check.get("producer"), record["target_binding"] + if not isinstance(producer, dict) or not isinstance(binding, dict): + return False + if (producer.get("kind") not in required["allowed_producer_kinds"] + or producer.get("control_domain") not in required["allowed_control_domains"] + or producer.get("control_domain") != binding.get("control_domain") + or not preparation._id(producer.get("capture_route"))): + return False + worker = producer["kind"] == "task_worker" + if worker or required["check_id"] == "samples": + runtimes = binding.get("runtime_bindings_by_step") + if (record["started"] is not True or not isinstance(record["candidate"], dict) + or not isinstance(runtimes, list)): + return False + recipe = prepared["recipe"] + step_ids = {step["step_id"] for step in record["steps"] if isinstance(step, dict) + and preparation._id(step.get("step_id")) + and step.get("started_at") is not None and step.get("status") != "not_started"} + input_steps = {step for item in record["bound_inputs"] if isinstance(item, dict) + and item.get("role") == record["candidate"]["input_role"] + and item.get("attempt_id") == record["attempt_id"] + and isinstance(item.get("step_ids"), list) + for step in item["step_ids"] if preparation._id(step)} + if not any(isinstance(runtime, dict) + and runtime.get("step_id") in step_ids & input_steps + and runtime.get("runtime_role") == recipe["runtime_role"] + and runtime.get("entrypoint") == recipe["entrypoint"] + and runtime.get("entrypoint_revision") == recipe["entrypoint_revision"] + and runtime.get("code_sha256") == recipe["code_sha256"] + for runtime in runtimes): + return False + if worker: + return (producer.get("identity") == recipe["entrypoint"] + and producer.get("revision") == recipe["entrypoint_revision"]) + return preparation._id(producer.get("identity")) + + +def _proof_json(data): + # Raw evidence is hashed as retained, not rewritten into the DTO encoding. + # JSON whitespace is allowed; duplicate keys, non-finite/floating tokens, + # invalid Unicode and out-of-range integers must still be refused. + def pairs(items): + result = {} + for key, item in items: + _require(key not in result, "Duplicate evidence key.") + result[key] = item + return result + + def number(_): + raise AssessmentError("InvalidRequest", "Evidence requires Decimal strings.") + + try: + value = json.loads(data.decode("utf-8"), object_pairs_hook=pairs, + parse_float=number, parse_constant=number) + canonical_encode(value) + except (ValueError, UnicodeError, RecursionError): + raise AssessmentError("InvalidRequest", "Invalid P1 evidence JSON.") from None + return value + + +def _proof(data, required, record, actual): + value = _proof_json(data) + key = required["check_id"] + common = {"schema_version": 1, "attempt_id": record["attempt_id"], + "operation": record["operation"], "validation_scope": record["validation_scope"]} + if key == "cleanup": + _object(value, set(common) | {"confirmed"}) + _require(type(value["confirmed"]) is bool) + else: + common.update(candidate_id=record["candidate"]["candidate_id"], actual_sha256=actual) + _require(actual is not None) + if key in ("format", "grader"): + _object(value, _COMMON | {"claim_id", "checker_revision", "status", "reasons"}) + _require(value["claim_id"] == required["for_claims"][0] + and value["checker_revision"] == required["checker_revision"] + and value["status"] in {"passed", "failed"}) + _strings(value["reasons"], identifiers=True) + else: + _object(value, _COMMON | {"samples"}) + _require(isinstance(value["samples"], list) and len(value["samples"]) == 4) + for sample, identity in zip(value["samples"], ( + ("A1", "baseline", 1), ("B1", "candidate", 1), + ("B2", "candidate", 2), ("A2", "baseline", 2), + )): + _object(sample, {"sample_id", "role", "round", "value", "unit"}) + _require((sample["sample_id"], sample["role"], sample["round"]) == identity + and type(sample["round"]) is int and sample["unit"] == "ms" + and isinstance(sample["value"], str) + and preparation._DECIMAL.fullmatch(sample["value"]) is not None + and Decimal(sample["value"]) > 0) + _require(type(value["schema_version"]) is int + and all(value[field] == expected for field, expected in common.items())) + return value + + +def _observations(required, prepared, record, actual, reader): + proofs, refs, problems = [], [], [] + for check in record["checks"]: + if not isinstance(check, dict) or check.get("check_id") != required["check_id"]: + continue + if (not _trusted(check, required, prepared, record) + or check.get("status") not in {"passed", "failed"} + or not isinstance(check.get("evidence_refs"), list)): + problems.append("untrusted_or_incomplete_check") + continue + for ref in check["evidence_refs"]: + data, reason = reader.read(ref) + if data is None: + problems.append(reason) + continue + try: + proof = _proof(data, required, record, actual) + if "status" in proof: + _require(proof["status"] == check["status"]) + except (CanonicalError, AssessmentError, KeyError, TypeError): + problems.append("invalid_evidence") + continue + proofs.append(proof) + refs.append(ref) + key = required["check_id"] + signatures = {canonical_encode(proof if key == "samples" else proof.get( + "status", proof.get("confirmed"))) for proof in proofs} + return proofs, refs, problems, len(signatures) > 1 + + +def _unique(refs): + return list({canonical_encode(ref): deepcopy(ref) for ref in refs}.values()) + + +def _cleanup_confirmed(record): + cleanup, resources = record["cleanup"], record["resource_leases"] + if (not isinstance(cleanup, dict) or not isinstance(resources, dict) + or not isinstance(resources.get("leases"), list)): + return False + if (cleanup.get("status") != "confirmed" or cleanup.get("process_exit_confirmed") is not True + or resources.get("applicability") not in {"applicable", "not_applicable"} + or resources.get("applicability") == "not_applicable" and resources["leases"] + or resources.get("applicability") == "applicable" and not resources["leases"]): + return False + for lease in resources["leases"]: + if not isinstance(lease, dict) or lease.get("release_confirmed") is not True: + return False + try: + _timestamp(lease.get("release_confirmed_at")) + except AssessmentError: + return False + return True + + +def _claim(definition, protocol, prepared, record, actual, reader): + missing, reasons, refs, decisions = [], [], [], [] + conflict = False + for required in protocol["required_evidence"]: + if definition["claim_id"] not in required["for_claims"]: + continue + proofs, used, problems, contradicted = _observations( + required, prepared, record, actual, reader, + ) + refs.extend(used) + conflict |= contradicted + if not proofs: + missing.append(required["check_id"]) + reasons.extend(problems) + elif required["check_id"] == "cleanup" and not all(p["confirmed"] for p in proofs): + missing.append("cleanup") + else: + decisions.extend(p.get("status", "passed") for p in proofs) + reasons.extend(reason for p in proofs for reason in p.get("reasons", [])) + if definition["requires_completed_measurement"] and not ( + record["phase"] == "finished" and record["started"] is True + and record["process_outcome"] == "exited" and record["exit_code"] == 0 + and record["side_effect_unknown"] is False + and record["steps"] and all(isinstance(step, dict) and step.get("status") == "exited" + and step.get("exit_code") == 0 for step in record["steps"]) + ): + missing.append("completed_measurement") + if definition["requires_cleanup_confirmed"] and not _cleanup_confirmed(record): + missing.append("cleanup_confirmation") + if conflict: + reasons.append("conflicting_evidence") + if missing: + reasons.append("required_evidence_missing") + verdict = "INCONCLUSIVE" if conflict or missing else ( + "FAIL" if "failed" in decisions else "PASS" + ) + return { + "claim_id": definition["claim_id"], "validation_scope": definition["validation_scope"], + "verdict": verdict, "reasons": list(dict.fromkeys(reasons)), "evidence": _unique(refs), + "missing_evidence": list(dict.fromkeys(missing)), "limitations": [], + } + + +class AssessmentService: + """The offline P1 port; a clock is injected only for reproducible timestamps.""" + + def __init__(self, *, clock=None): + self._clock = clock or ( + lambda: datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + ) + + def assess(self, prepared: dict, evidence: dict, *, artifacts) -> dict: + try: + prepared = canonical_decode(canonical_encode(prepared)) + evidence = canonical_decode(canonical_encode(evidence)) + try: + record = _binding(prepared, evidence) + except (KeyError, TypeError): + raise AssessmentError("InvalidRequest", "Malformed fixed evidence input.") from None + reader = _RetainedBytes(artifacts) + data, reason = reader.read(prepared["protocol"]) + _require(data is not None, "Frozen protocol unavailable.", + reason if reason in _ITEM_ERRORS else "InvalidRequest") + preparation.validate_prepared(prepared, data) + protocol = canonical_decode(data) + actual = _captured_candidate(prepared, record, reader) + claims = [_claim(item, protocol, prepared, record, actual, reader) + for item in protocol["claim_definitions"]] + verdicts = {claim["verdict"] for claim in claims} + conflict = any("conflicting_evidence" in claim["reasons"] for claim in claims) + status = "ineligible" if conflict or "FAIL" in verdicts else ( + "eligible" if verdicts == {"PASS"} else "unknown" + ) + result = { + "schema_version": 1, "assessment_id": "", "task": deepcopy(prepared["task"]), + "grader": deepcopy(prepared["grader"]), + "assessor": {"name": "fixture-assessor", "revision": "1"}, + "prepared_ref": deepcopy(record["prepared_ref"]), + "record_ref": deepcopy(evidence["record_ref"]), + "protocol_digest": record["protocol_digest"], + "candidate": deepcopy(record["candidate"]), "attempt_id": record["attempt_id"], + "claims": claims, "metrics": [], "comparison": [], + "eligibility": [{"validation_scope": "p1", "status": status, + "reasons": ["conflicting_evidence"] if conflict else ( + ["claim_failed"] if status == "ineligible" else ( + [] if status == "eligible" else ["incomplete_evidence"] + ))}], + "evidence_artifacts": reader.verified, "assessed_at": self._clock(), + "supersedes": None, "derived_from": [], "limitations": [], + } + result["assessment_id"] = assessment_id(result) + validate_assessment(result) + return result + except (AssessmentError, preparation.PreparationError) as error: + return AssessmentError(error.code, str(error)).to_json() + except CanonicalError: + return AssessmentError( + "InvalidRequest", "Malformed canonical assessment input.", + ).to_json() + + +def assess(prepared: dict, evidence: dict, *, artifacts) -> dict: + """Assess the first frozen task using a real UTC assessment clock.""" + return AssessmentService().assess(prepared, evidence, artifacts=artifacts) diff --git a/tests/test_benchmark_assessment.py b/tests/test_benchmark_assessment.py new file mode 100644 index 00000000..9d7ad10a --- /dev/null +++ b/tests/test_benchmark_assessment.py @@ -0,0 +1,539 @@ +"""The production offline port over retained P1 fixtures; no evaluation worker is run.""" + +import json +import subprocess +import sys +from copy import deepcopy +from hashlib import sha256 +from io import BytesIO +from itertools import count +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from inferrsi.apps import api as apps_api +from inferrsi.apps.handoff.bridge import check_assessment_identity, to_evidence +from inferrsi.apps.storage.archive import read_prepared +from inferrsi.apps.storage.artifact_reader import ArtifactReader +from inferrsi.benchmark import assessment +from inferrsi.benchmark import preparation as preparation_port +from inferrsi.contracts.json_v1 import decode, encode +from inferrsi.harness.errors import PublicError +from tests.test_benchmark_preparation import preparation as preparation + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = ROOT / "docs/contracts/fixtures/t108-v4" +NOW = "2099-01-01T00:00:00Z" + + +def test_pure_codec_preserves_the_frozen_documentary_result(monkeypatch): + data = (GOLDEN / "assessment.json").read_bytes() + result = decode(data) + monkeypatch.setattr(Path, "open", lambda *a, **k: pytest.fail("pure codec opened a file")) + assert assessment.encode(result) == data + assert assessment.decode(data) == result + assessment.validate_assessment(result) + check_assessment_identity(result) + + +@pytest.fixture +def retained(preparation, tmp_path): + service, request, _, _, _, config, directory = preparation + output = apps_api.prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=preparation_port.encode, + validate_prepared=preparation_port.validate_prepared, + ) + assert output.result["exit_code"] == 0, output.result + prepared_ref = output.result["prepared_ref"] + prepared = read_prepared(config, prepared_ref) + record = decode((GOLDEN / "record-payload.json").read_bytes()) + record.update( + prepared_ref=prepared_ref, protocol_digest=prepared["protocol_digest"], + started=True, phase="finished", process_outcome="exited", exit_code=0, + elapsed_seconds="1", checks=[], artifacts=[], errors=[], + cleanup={"status": "confirmed", "process_exit_confirmed": True, + "termination_requested": False, "reasons": []}, + ) + record["steps"][0].update(status="exited", started_at=NOW, ended_at=NOW, + elapsed_seconds="1", exit_code=0, reasons=[]) + record["target_binding"] = { + "target_id": record["target_request"]["target_id"], "config_revision": "1", + "executor_version": "1", "control_domain": "fixture-domain", "capability_checks": [], + "runtime_bindings_by_step": [{ + "step_id": "main", "runtime_role": "local-python", "executable": "python", + "entrypoint": prepared["recipe"]["entrypoint"], "entrypoint_revision": "1", + "code_sha256": prepared["recipe"]["code_sha256"], "argv": [], + "environment_identity": None, + }], + } + serial = count() + + def put(label, data): + name = f"retained/{next(serial)}-{label}" + path = directory / name + path.parent.mkdir(exist_ok=True) + path.write_bytes(data) + return {"location": "archives/local/" + name, "sha256": sha256(data).hexdigest(), + "size_bytes": len(data), "media_type": "application/json"} + + candidate = put("candidate.txt", b"retained software-test candidate\n") + record["bound_inputs"] = [{ + "role": "candidate", "retained_ref": candidate, "actual_sha256": candidate["sha256"], + "snapshot_format": "file-v1", "captured_at": NOW, "attempt_id": record["attempt_id"], + "step_ids": ["main"], "materialized_path": "inputs/candidate.txt", + "expected_sha256": candidate["sha256"], "identity_status": "matched", "reasons": [], + }] + + def add(key, status="passed", **changes): + requirement = next(x for x in prepared["required_evidence"] if x["check_id"] == key) + proof = {"schema_version": 1, "attempt_id": record["attempt_id"], + "operation": "evaluate", "validation_scope": "p1"} + if key != "cleanup": + proof.update(candidate_id=record["candidate"]["candidate_id"], + actual_sha256=candidate["sha256"]) + if key in {"format", "grader"}: + proof.update(claim_id=requirement["for_claims"][0], checker_revision="1", + status=status, reasons=[] if status == "passed" else ["fixture_rejection"]) + elif key == "samples": + proof["samples"] = [ + {"sample_id": name, "role": role, "round": round_, "value": "1", "unit": "ms"} + for name, role, round_ in [("A1", "baseline", 1), ("B1", "candidate", 1), + ("B2", "candidate", 2), ("A2", "baseline", 2)] + ] + else: + proof["confirmed"] = True + proof.update(changes) + ref = put(key + ".json", encode(proof)) + kind = requirement["allowed_producer_kinds"][0] + check = { + **{k: requirement[k] for k in ("check_id", "checker_revision", "subject", "phase", + "artifact_schema")}, + "attempt_id": record["attempt_id"], "operation": "evaluate", "validation_scope": "p1", + "producer": {"kind": kind, "identity": "fixture_worker" if kind == "task_worker" + else "fixture-harness", "revision": "1", + "control_domain": "fixture-domain", + "capture_route": "retained-software-fixture"}, + "status": status, "evidence_refs": [ref], "reasons": [], + } + record["checks"].append(check) + return check, proof + + for key in ("format", "grader", "samples", "cleanup"): + add(key) + + def seal(): + ref = {"attempt_id": record["attempt_id"], "record_version": record["record_version"], + "artifact": put("record.json", encode(record))} + return ref, to_evidence(prepared, prepared_ref, ref, record) + + return {"prepared": prepared, "record": record, "config": config, "directory": directory, + "put": put, "add": add, "seal": seal, "workspace": tmp_path} + + +def run(case, *, reader=None, clock=lambda: NOW): + _, evidence = case["seal"]() + result = assessment.AssessmentService(clock=clock).assess( + case["prepared"], evidence, artifacts=reader or ArtifactReader(case["config"]), + ) + return result + + +def remove(case, *keys): + case["record"]["checks"] = [c for c in case["record"]["checks"] if c["check_id"] not in keys] + + +def test_all_positive_proofs_are_validated_against_captured_bytes(retained): + result = run(retained) + assert "code" not in result, result + assert [c["verdict"] for c in result["claims"]] == ["PASS"] * 3 + assert result["eligibility"][0]["status"] == "eligible" + assessment.validate_assessment(result) + check_assessment_identity(result) + assert assessment.decode(assessment.encode(result)) == result + schema = json.loads((ROOT / "docs/contracts/schemas/contract-v1.schema.json").read_text()) + Draft202012Validator({**schema, "$ref": "#/$defs/EvaluationResult"}).validate(result) + _, evidence = retained["seal"]() + Draft202012Validator({**schema, "$ref": "#/$defs/EvidenceInput"}).validate(evidence) + + +def test_t35_three_independent_claims_and_apps_exit_codes(retained): + remove(retained, "grader", "samples", "cleanup") + retained["add"]("grader", "failed") + ref, _ = retained["seal"]() + service = assessment.AssessmentService(clock=lambda: NOW) + for claim, exit_code in (("F", 0), ("C", 8), ("M", 9)): + workspace = retained["workspace"] / claim + workspace.mkdir() + output = apps_api.assess( + {"schema_version": 1, "record_ref": ref, + "primary_scope": "p1", "primary_claim_id": claim}, + workspace=workspace, storage_config=retained["config"], + assess_benchmark=service.assess, validate_assessment=assessment.validate_assessment, + encode_assessment=assessment.encode, + ) + assert output.result["exit_code"] == exit_code, output.result + assert output.result["assessment_ref"] is not None + assert [c["verdict"] for c in output.result["assessment"]["claims"]] == [ + "PASS", "FAIL", "INCONCLUSIVE", + ] + + +@pytest.mark.parametrize("kind", ["UnsafeArtifactPath", "ArtifactUnavailable", + "ArtifactIntegrityError", "bad_bytes"]) +def test_each_read_failure_is_only_a_claim_gap(retained, kind): + target = retained["record"]["checks"][0]["evidence_refs"][0] + reader = ArtifactReader(retained["config"]) + + class Fault: + def open(self, ref): + if ref == target: + if kind == "bad_bytes": + return BytesIO(b"not the retained bytes") + raise PublicError(kind, stage="open", message="fixture rejection") + return reader.open(ref) + + result = run(retained, reader=Fault()) + assert [c["verdict"] for c in result["claims"]] == ["INCONCLUSIVE", "PASS", "PASS"] + assert result["claims"][0]["missing_evidence"] == ["format"] + + +def test_host_programming_errors_are_not_disguised_as_missing_evidence(retained): + class Broken: + def open(self, ref): + raise TypeError("broken host adapter") + + with pytest.raises(TypeError, match="broken host"): + run(retained, reader=Broken()) + + +@pytest.mark.parametrize("field,value", [("kind", "caller_assertion"), + ("control_domain", "untrusted-domain")]) +def test_untrusted_self_assertions_cannot_pass(retained, field, value): + retained["record"]["checks"][0]["producer"][field] = value + assert run(retained)["claims"][0]["verdict"] == "INCONCLUSIVE" + + +def test_proof_candidate_hash_is_not_its_own_identity_authority(retained): + remove(retained, "format") + retained["add"]("format", actual_sha256="0" * 64) + assert run(retained)["claims"][0]["verdict"] == "INCONCLUSIVE" + + +def test_v21_conflicting_proofs_are_aggregated_once(retained): + retained["add"]("format", "failed") + result = run(retained) + assert len(result["claims"]) == 3 + assert result["claims"][0]["verdict"] == "INCONCLUSIVE" + assert "conflicting_evidence" in result["claims"][0]["reasons"] + assert result["eligibility"][0]["status"] == "ineligible" + + +def test_running_snapshot_keeps_missing_terminal_evidence_inconclusive(retained): + remove(retained, "grader", "samples", "cleanup") + retained["record"].update(phase="running", process_outcome="unknown", exit_code=None) + retained["record"]["cleanup"].update(status="unknown", process_exit_confirmed=None) + before = encode(retained["record"]) + result = run(retained) + assert [c["verdict"] for c in result["claims"]] == ["PASS", "INCONCLUSIVE", "INCONCLUSIVE"] + assert result["claims"][1]["missing_evidence"] == ["grader"] + assert "completed_measurement" in result["claims"][2]["missing_evidence"] + assert encode(retained["record"]) == before + + +@pytest.mark.parametrize("mutation", ["order", "zero", "unit", "missing", "cleanup", "lease"]) +def test_measurement_requires_complete_sampling_cleanup_and_release(retained, mutation): + if mutation in {"order", "zero", "unit", "missing"}: + old = next(c for c in retained["record"]["checks"] if c["check_id"] == "samples") + with ArtifactReader(retained["config"]).open(old["evidence_refs"][0]) as stream: + samples = decode(stream.read())["samples"] + remove(retained, "samples") + if mutation == "order": + samples.reverse() + elif mutation == "zero": + samples[0]["value"] = "0" + elif mutation == "unit": + samples[0]["unit"] = "second" + else: + samples.pop() + retained["add"]("samples", samples=samples) + elif mutation == "cleanup": + retained["record"]["cleanup"]["process_exit_confirmed"] = None + else: + retained["record"]["resource_leases"] = { + "applicability": "applicable", "reasons": [], "leases": [{ + "lease_id": "lease-1", "resource_id": "cpu", "control_domain": "fixture-domain", + "owner": "fixture", "acquired_at": NOW, "release_requested_at": NOW, + "release_confirmed": False, "release_confirmed_at": None, "reasons": [], + }], + } + assert run(retained)["claims"][2]["verdict"] == "INCONCLUSIVE" + + +def test_binding_mismatch_is_refused_before_any_artifact_open(retained): + _, evidence = retained["seal"]() + evidence["record_digest"] = "0" * 64 + + class Never: + def open(self, ref): + pytest.fail("identity mismatch reached artifact IO") + + result = assessment.assess(retained["prepared"], evidence, artifacts=Never()) + assert result["code"] == "IdentityMismatch" + assert result["stage"] == "assessment_binding" + assert "assessment_id" not in result + + +@pytest.mark.parametrize("revision", ["grader", "claim-rule"]) +def test_v22_unsupported_same_family_rule_revision_is_refused(retained, revision): + prepared = retained["prepared"] + reader = ArtifactReader(retained["config"]) + with reader.open(prepared["protocol"]) as stream: + protocol = decode(stream.read()) + if revision == "grader": + protocol["grader"]["revision"] = "2" + prepared["grader"]["revision"] = "2" + else: + protocol["claim_definitions"][0]["decision_rule"]["revision"] = "2" + prepared["protocol"] = retained["put"]("unsupported-protocol.json", encode(protocol)) + prepared["protocol_digest"] = preparation_port.protocol_digest(protocol) + prepared["prepared_id"] = preparation_port.prepared_id(prepared) + record = retained["record"] + record["protocol_digest"] = prepared["protocol_digest"] + record["prepared_ref"]["prepared_id"] = prepared["prepared_id"] + record["prepared_ref"]["artifact"] = retained["put"]( + "unsupported-prepared.json", encode(prepared), + ) + result = run(retained) + assert result["code"] == "UnsupportedRuleVersion" + + +def test_outer_prepared_relocation_keeps_semantic_binding(retained): + before = retained["prepared"]["prepared_id"] + reader = ArtifactReader(retained["config"]) + ref = retained["prepared"]["protocol"] + with reader.open(ref) as stream: + ref["location"] = retained["put"]("relocated-protocol.json", stream.read())["location"] + assert preparation_port.prepared_id(retained["prepared"]) == before + retained["record"]["prepared_ref"]["artifact"] = retained["put"]( + "relocated-prepared.json", encode(retained["prepared"]), + ) + assert all(c["verdict"] == "PASS" for c in run(retained)["claims"]) + + +def test_v23_clock_and_typed_location_projection(retained): + result = run(retained) + later = run(retained, clock=lambda: "2099-01-01T00:00:01Z") + # Each seal creates a new locator but has identical fixed payload bytes. + assert result["assessment_id"] == later["assessment_id"] + assert assessment.encode(result) != assessment.encode(later) + changed = deepcopy(result) + changed["record_ref"]["artifact"]["location"] = "elsewhere/record.json" + check_assessment_identity(changed) + assessment.validate_assessment(changed) + changed["claims"][0]["verdict"] = "FAIL" + with pytest.raises(ValueError): + assessment.validate_assessment(changed) + + +def test_v23_fixed_clock_is_byte_identical_in_an_isolated_process(retained, tmp_path): + _, evidence = retained["seal"]() + reader = ArtifactReader(retained["config"]) + expected = assessment.AssessmentService(clock=lambda: NOW).assess( + retained["prepared"], evidence, artifacts=reader, + ) + blobs = {} + for ref in expected["evidence_artifacts"]: + with reader.open(ref) as stream: + blobs[ref["location"]] = stream.read().hex() + case = tmp_path / "case.json" + case.write_bytes(encode({ + "prepared": retained["prepared"], "evidence": evidence, "blobs": blobs, + })) + child = """ +import sys +from io import BytesIO +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from inferrsi.benchmark.assessment import AssessmentService, encode +from inferrsi.contracts.json_v1 import decode +value = decode(Path(sys.argv[2]).read_bytes()) +class Reader: + def open(self, ref): + return BytesIO(bytes.fromhex(value['blobs'][ref['location']])) +result = AssessmentService(clock=lambda: '2099-01-01T00:00:00Z').assess( + value['prepared'], value['evidence'], artifacts=Reader()) +sys.stdout.buffer.write(encode(result)) +""" + done = subprocess.run([sys.executable, "-I", "-c", child, str(ROOT / "src"), str(case)], + cwd=tmp_path, capture_output=True, check=True, timeout=30) + assert done.stdout == assessment.encode(expected) + + +@pytest.mark.parametrize("primary", [None, "unknown", "missing"]) +def test_t39_bad_primary_is_rejected_before_the_assessor(retained, primary): + ref, _ = retained["seal"]() + + def never(*args, **kwargs): + pytest.fail("bad primary reached Benchmark") + + request = {"schema_version": 1, "record_ref": ref, + "primary_scope": "p1", "primary_claim_id": primary} + if primary == "missing": + del request["primary_claim_id"] + result = apps_api.assess( + request, + workspace=retained["workspace"], storage_config=retained["config"], + assess_benchmark=never, validate_assessment=assessment.validate_assessment, + encode_assessment=assessment.encode, + ).result + assert result["exit_code"] == 2 and result["reason_code"] == "InvalidRequest" + + +def test_t39_missing_known_claim_and_failed_archive(retained, monkeypatch): + ref, _ = retained["seal"]() + request = {"schema_version": 1, "record_ref": ref, + "primary_scope": "p1", "primary_claim_id": "F"} + service = assessment.AssessmentService(clock=lambda: NOW) + + def omitted(*args, **kwargs): + result = service.assess(*args, **kwargs) + result["claims"].pop(0) + return result + + options = {"workspace": retained["workspace"], "storage_config": retained["config"], + "validate_assessment": assessment.validate_assessment, + "encode_assessment": assessment.encode} + result = apps_api.assess(request, assess_benchmark=omitted, **options).result + assert result["exit_code"] == 2 and result["reason_code"] == "AssessorError" + assert result["assessment_ref"] is None + + def failed(*args, **kwargs): + raise PublicError( + "ArchivePersistenceError", stage="assessment_archive", message="fixture IO", + ) + + monkeypatch.setattr(apps_api, "archive_assessment", failed) + result = apps_api.assess(request, assess_benchmark=service.assess, **options).result + assert result["exit_code"] == 6 and result["assessment_ref"] is None + assert result["assessment"] is not None + + +def test_assessment_does_not_launch_workers_or_access_the_network(retained, monkeypatch): + import socket + + def forbidden(*args, **kwargs): + pytest.fail("offline assessor attempted execution or network access") + + monkeypatch.setattr(subprocess, "Popen", forbidden) + monkeypatch.setattr(socket, "socket", forbidden) + assert all(c["verdict"] == "PASS" for c in run(retained)["claims"]) + + +@pytest.mark.parametrize("source", ["wrong-code", "foreign-check", "uncaptured", "not-started"]) +def test_evidence_cannot_override_actual_runtime_and_input_capture(retained, source): + record = retained["record"] + if source == "wrong-code": + record["target_binding"]["runtime_bindings_by_step"][0]["code_sha256"] = "0" * 64 + expected = ["PASS", "INCONCLUSIVE", "INCONCLUSIVE"] + elif source == "foreign-check": + record["checks"][0]["attempt_id"] = "foreign-attempt" + expected = ["INCONCLUSIVE", "PASS", "PASS"] + elif source == "uncaptured": + record["bound_inputs"][0]["actual_sha256"] = "0" * 64 + expected = ["INCONCLUSIVE"] * 3 + else: + record.update(started=False, process_outcome="not_started", exit_code=None) + record["steps"][0].update(status="not_started", started_at=None, ended_at=None, + exit_code=None, elapsed_seconds=None) + expected = ["PASS", "INCONCLUSIVE", "INCONCLUSIVE"] + assert [c["verdict"] for c in run(retained)["claims"]] == expected + + +def test_conflicting_measurements_are_not_first_or_last_wins(retained): + first = next(c for c in retained["record"]["checks"] if c["check_id"] == "samples") + with ArtifactReader(retained["config"]).open(first["evidence_refs"][0]) as stream: + samples = decode(stream.read())["samples"] + samples[0]["value"] = "2" + retained["add"]("samples", samples=samples) + result = run(retained) + assert result["claims"][2]["verdict"] == "INCONCLUSIVE" + assert "conflicting_evidence" in result["claims"][2]["reasons"] + assert result["eligibility"][0]["status"] == "ineligible" + + +def test_typed_projection_is_independent_of_python_object_aliasing(retained): + result = run(retained) + shared = result["claims"][0]["evidence"][0] + result["evidence_artifacts"].append(shared) + result["candidate"]["context_refs"] = [shared] + result["assessment_id"] = assessment.assessment_id(result) + assessment.validate_assessment(result) + check_assessment_identity(result) + + +@pytest.mark.parametrize("mutation", ["duplicate", "unknown-claim", "extra-field", "metric"]) +def test_pure_validator_rejects_invalid_results_even_with_recomputed_identity(retained, mutation): + result = run(retained) + if mutation == "duplicate": + result["claims"][1] = deepcopy(result["claims"][0]) + elif mutation == "unknown-claim": + result["claims"][0]["claim_id"] = "unknown" + elif mutation == "extra-field": + result["claims"][0]["overall"] = "PASS" + else: + result["metrics"] = [{"raw_samples": []}] + result["assessment_id"] = assessment.assessment_id(result) + with pytest.raises(ValueError): + assessment.validate_assessment(result) + + +def test_pure_timestamp_validation_checks_calendar_and_preserves_fractional_precision(retained): + result = run(retained) + result["assessed_at"] = "2099-02-29T00:00:00Z" + with pytest.raises(ValueError): + assessment.validate_assessment(result) + result["assessed_at"] = "2096-02-29T00:00:00.123456789Z" + assert assessment.decode(assessment.encode(result)) == result + + +@pytest.mark.parametrize("duplicate", [False, True]) +def test_raw_evidence_keeps_original_bytes_and_rejects_duplicate_json_keys(retained, duplicate): + check = retained["record"]["checks"][0] + with ArtifactReader(retained["config"]).open(check["evidence_refs"][0]) as stream: + proof = decode(stream.read()) + text = json.dumps(proof, indent=2) + if duplicate: + text = text[:-1] + ', "status": "passed"}' + raw = text.encode("utf-8") + check["evidence_refs"] = [retained["put"]("noncanonical-proof.json", raw)] + result = run(retained) + assert result["claims"][0]["verdict"] == ("INCONCLUSIVE" if duplicate else "PASS") + + +@pytest.mark.parametrize("character", ["\x00", "\x1f", "\x7f"]) +def test_pure_codec_rejects_control_characters_in_typed_paths(retained, character): + result = run(retained) + result["record_ref"]["artifact"]["location"] = "invalid/" + character + "/record.json" + with pytest.raises(ValueError): + assessment.encode(result) + + +def test_host_reader_cannot_change_the_snapshotted_input_during_assessment(retained): + _, evidence = retained["seal"]() + reader = ArtifactReader(retained["config"]) + digest = retained["prepared"]["protocol_digest"] + + class Mutating: + def open(self, ref): + retained["prepared"]["protocol_digest"] = "sha256:" + "0" * 64 + evidence["record"]["phase"] = "running" + return reader.open(ref) + + result = assessment.AssessmentService(clock=lambda: NOW).assess( + retained["prepared"], evidence, artifacts=Mutating(), + ) + assert result["protocol_digest"] == digest + assert all(c["verdict"] == "PASS" for c in result["claims"]) From 1565602d4f6805ea2b366b22b65c5e86d3e8783b Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 19:00:41 +0800 Subject: [PATCH 2/5] test(benchmark): verify offline port in isolated installations Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- tests/check_installed.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/check_installed.py b/tests/check_installed.py index 5b02b0d9..1fc9aed9 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -791,6 +791,23 @@ def _inventory(root, prefix): [sys.executable, "-I", "-c", facade_probe], capture_output=True, text=True, check=False, ) assert facade.returncode == 0, facade.stderr +# The offline Benchmark port must ship in the wheel and remain independently importable. +assessment_probe = ( + "import sys\n" + "from inferrsi.benchmark.assessment import (\n" + " AssessmentService, assess, decode, encode, validate_assessment,\n" + ")\n" + "assert all(callable(port) for port in\n" + " (AssessmentService, assess, decode, encode, validate_assessment))\n" + "for prefix in ('inferrsi.apps', 'inferrsi.harness', 'inferrsi.knowledge'):\n" + " assert not [name for name in sys.modules if name == prefix\n" + " or name.startswith(prefix + '.')], prefix\n" +) +assessment_port = subprocess.run( + [sys.executable, "-I", "-B", "-c", assessment_probe], + capture_output=True, text=True, check=False, +) +assert assessment_port.returncode == 0, assessment_port.stderr # #452 M1: the MindIE adapter ships in the wheel. A fresh isolated interpreter imports the # package initializer with no core loaded and without a re-exported version, then imports # the stdio MCP module through its absolute package path. From 51ae6d011d89bbdbf67657716de51f551ded1096 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 19:05:20 +0800 Subject: [PATCH 3/5] fix(benchmark): preserve unknown clocks and validate resource locators Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- src/inferrsi/benchmark/assessment.py | 9 ++++++++- tests/test_benchmark_assessment.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/inferrsi/benchmark/assessment.py b/src/inferrsi/benchmark/assessment.py index 7989dacd..3c90f7bc 100644 --- a/src/inferrsi/benchmark/assessment.py +++ b/src/inferrsi/benchmark/assessment.py @@ -233,6 +233,9 @@ def read(self, ref): def _binding(prepared, evidence): preparation.encode(prepared) + _artifact(prepared["protocol"]) + for resource in prepared["resources"]: + _artifact(resource["artifact"]) _object(evidence, {"schema_version", "record_ref", "record_digest", "task", "task_source", "record", "artifact_checks"}) record, ref = evidence["record"], evidence["record_ref"] @@ -313,7 +316,9 @@ def _trusted(check, required, prepared, record): recipe = prepared["recipe"] step_ids = {step["step_id"] for step in record["steps"] if isinstance(step, dict) and preparation._id(step.get("step_id")) - and step.get("started_at") is not None and step.get("status") != "not_started"} + and step.get("status") != "not_started" + and (step.get("started_at") is not None or step.get("status") + in {"running", "exited", "timed_out", "cancelled"})} input_steps = {step for item in record["bound_inputs"] if isinstance(item, dict) and item.get("role") == record["candidate"]["input_role"] and item.get("attempt_id") == record["attempt_id"] @@ -510,6 +515,8 @@ def assess(self, prepared: dict, evidence: dict, *, artifacts) -> dict: reason if reason in _ITEM_ERRORS else "InvalidRequest") preparation.validate_prepared(prepared, data) protocol = canonical_decode(data) + for resource in preparation._closure(protocol): + _artifact(resource["artifact"]) actual = _captured_candidate(prepared, record, reader) claims = [_claim(item, protocol, prepared, record, actual, reader) for item in protocol["claim_definitions"]] diff --git a/tests/test_benchmark_assessment.py b/tests/test_benchmark_assessment.py index 9d7ad10a..ba4821b6 100644 --- a/tests/test_benchmark_assessment.py +++ b/tests/test_benchmark_assessment.py @@ -537,3 +537,21 @@ def open(self, ref): ) assert result["protocol_digest"] == digest assert all(c["verdict"] == "PASS" for c in result["claims"]) + + +def test_confirmed_execution_does_not_require_fabricating_unknown_clock_facts(retained): + retained["record"]["steps"][0].update(started_at=None, ended_at=None) + assert all(c["verdict"] == "PASS" for c in run(retained)["claims"]) + assert retained["record"]["steps"][0]["started_at"] is None + + +def test_invalid_prepared_resource_path_is_rejected_without_reading(retained): + _, evidence = retained["seal"]() + retained["prepared"]["resources"][0]["artifact"]["location"] = "invalid/\x00/resource" + + class Never: + def open(self, ref): + pytest.fail("invalid input reached artifact IO") + + result = assessment.assess(retained["prepared"], evidence, artifacts=Never()) + assert result["code"] == "InvalidRequest" From 4258c76a84f3c9cf7086a0b7965a7ec17cd445f9 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 19:13:09 +0800 Subject: [PATCH 4/5] fix(benchmark): separate host failures from input rejections Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- src/inferrsi/benchmark/assessment.py | 22 ++++++++++++++++------ tests/test_benchmark_assessment.py | 28 +++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/inferrsi/benchmark/assessment.py b/src/inferrsi/benchmark/assessment.py index 3c90f7bc..291a62eb 100644 --- a/src/inferrsi/benchmark/assessment.py +++ b/src/inferrsi/benchmark/assessment.py @@ -268,6 +268,12 @@ def _binding(prepared, evidence): and record["candidate"]["candidate_id"] == prepared["expected_candidate"]["candidate_id"], "Declared candidate identity differs.", "IdentityMismatch") + for item in record["bound_inputs"]: + if (isinstance(item, dict) and item.get("role") == record["candidate"]["input_role"] + and item.get("attempt_id") == record["attempt_id"] + and item.get("expected_sha256") is not None): + _require(item["expected_sha256"] == prepared["expected_candidate"]["sha256"], + "Declared candidate expectation differs.", "IdentityMismatch") return record @@ -505,6 +511,11 @@ def assess(self, prepared: dict, evidence: dict, *, artifacts) -> dict: try: prepared = canonical_decode(canonical_encode(prepared)) evidence = canonical_decode(canonical_encode(evidence)) + except CanonicalError: + return AssessmentError( + "InvalidRequest", "Malformed canonical assessment input.", + ).to_json() + try: try: record = _binding(prepared, evidence) except (KeyError, TypeError): @@ -513,8 +524,11 @@ def assess(self, prepared: dict, evidence: dict, *, artifacts) -> dict: data, reason = reader.read(prepared["protocol"]) _require(data is not None, "Frozen protocol unavailable.", reason if reason in _ITEM_ERRORS else "InvalidRequest") - preparation.validate_prepared(prepared, data) - protocol = canonical_decode(data) + try: + preparation.validate_prepared(prepared, data) + protocol = canonical_decode(data) + except CanonicalError: + raise AssessmentError("InvalidRequest", "Invalid frozen protocol bytes.") from None for resource in preparation._closure(protocol): _artifact(resource["artifact"]) actual = _captured_candidate(prepared, record, reader) @@ -547,10 +561,6 @@ def assess(self, prepared: dict, evidence: dict, *, artifacts) -> dict: return result except (AssessmentError, preparation.PreparationError) as error: return AssessmentError(error.code, str(error)).to_json() - except CanonicalError: - return AssessmentError( - "InvalidRequest", "Malformed canonical assessment input.", - ).to_json() def assess(prepared: dict, evidence: dict, *, artifacts) -> dict: diff --git a/tests/test_benchmark_assessment.py b/tests/test_benchmark_assessment.py index ba4821b6..83fdf210 100644 --- a/tests/test_benchmark_assessment.py +++ b/tests/test_benchmark_assessment.py @@ -18,7 +18,7 @@ from inferrsi.apps.storage.artifact_reader import ArtifactReader from inferrsi.benchmark import assessment from inferrsi.benchmark import preparation as preparation_port -from inferrsi.contracts.json_v1 import decode, encode +from inferrsi.contracts.json_v1 import CanonicalError, decode, encode from inferrsi.harness.errors import PublicError from tests.test_benchmark_preparation import preparation as preparation @@ -153,7 +153,9 @@ def test_all_positive_proofs_are_validated_against_captured_bytes(retained): assessment.validate_assessment(result) check_assessment_identity(result) assert assessment.decode(assessment.encode(result)) == result - schema = json.loads((ROOT / "docs/contracts/schemas/contract-v1.schema.json").read_text()) + schema = json.loads((ROOT / "docs/contracts/schemas/contract-v1.schema.json").read_text( + encoding="utf-8", + )) Draft202012Validator({**schema, "$ref": "#/$defs/EvaluationResult"}).validate(result) _, evidence = retained["seal"]() Draft202012Validator({**schema, "$ref": "#/$defs/EvidenceInput"}).validate(evidence) @@ -200,12 +202,15 @@ def open(self, ref): assert result["claims"][0]["missing_evidence"] == ["format"] -def test_host_programming_errors_are_not_disguised_as_missing_evidence(retained): +@pytest.mark.parametrize("error", [TypeError("broken host adapter"), + CanonicalError("host_adapter", "broken host adapter")], + ids=["type", "canonical"]) +def test_host_programming_errors_are_not_disguised_as_missing_evidence(retained, error): class Broken: def open(self, ref): - raise TypeError("broken host adapter") + raise error - with pytest.raises(TypeError, match="broken host"): + with pytest.raises(type(error), match="broken host"): run(retained, reader=Broken()) @@ -555,3 +560,16 @@ def open(self, ref): result = assessment.assess(retained["prepared"], evidence, artifacts=Never()) assert result["code"] == "InvalidRequest" + + +def test_conflicting_declared_candidate_expectation_is_a_binding_error(retained): + prepared, record = retained["prepared"], retained["record"] + prepared["expected_candidate"] = { + "candidate_id": record["candidate"]["candidate_id"], "sha256": "0" * 64, + "format_schema": deepcopy(prepared["candidate_contract"]["format_schema"]), + } + prepared["prepared_id"] = preparation_port.prepared_id(prepared) + record["prepared_ref"].update(prepared_id=prepared["prepared_id"], artifact=retained["put"]( + "different-expectation.json", preparation_port.encode(prepared), + )) + assert run(retained)["code"] == "IdentityMismatch" From 5c23d14ef2b198a3b3e9734b62912c372bfcae29 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 19:36:41 +0800 Subject: [PATCH 5/5] fix(benchmark): preserve every declared evidence gap Keep an affected claim inconclusive when any matching declared proof is unusable, even if another proof is valid. Return declared errors for malformed identity projections. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/production-assess.md | 4 ++ src/inferrsi/benchmark/assessment.py | 12 ++++-- tests/test_benchmark_assessment.py | 57 ++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/docs/contracts/production-assess.md b/docs/contracts/production-assess.md index 834f8bbc..a28c26b0 100644 --- a/docs/contracts/production-assess.md +++ b/docs/contracts/production-assess.md @@ -40,6 +40,10 @@ streams it opens. Portable `UnsafeArtifactPath`, `ArtifactUnavailable` and `ArtifactIntegrityError` exceptions are recognized by their exact `code`, without depending on another module's exception class. An affected proof becomes a claim-level evidence gap. Unknown host programming exceptions propagate. +Every declared proof for a matching check must be usable: a good proof does not +hide an unreadable, corrupt or invalid second proof, or another matching check +with no evidence. That claim remains INCONCLUSIVE, preserving the verified refs +and the gap reason, regardless of proof/check order. Other claims are independent. Failure to obtain the protocol itself is a port error: missing rules cannot be replaced by a guessed assessment. diff --git a/src/inferrsi/benchmark/assessment.py b/src/inferrsi/benchmark/assessment.py index 291a62eb..c04e376b 100644 --- a/src/inferrsi/benchmark/assessment.py +++ b/src/inferrsi/benchmark/assessment.py @@ -104,7 +104,12 @@ def _projection(result): def assessment_id(result: dict) -> str: """Pure v1 identity; only assessed_at, the self ID and typed locations are excluded.""" _object(result, _RESULT) - return "sha256:" + sha256(canonical_encode(_projection(result))).hexdigest() + try: + return "sha256:" + sha256(canonical_encode(_projection(result))).hexdigest() + except (KeyError, TypeError, CanonicalError, preparation.PreparationError): + raise AssessmentError( + "InvalidRequest", "Malformed assessment identity projection.", + ) from None def validate_assessment(result: dict) -> None: @@ -409,7 +414,8 @@ def _observations(required, prepared, record, actual, reader): continue if (not _trusted(check, required, prepared, record) or check.get("status") not in {"passed", "failed"} - or not isinstance(check.get("evidence_refs"), list)): + or not isinstance(check.get("evidence_refs"), list) + or not check["evidence_refs"]): problems.append("untrusted_or_incomplete_check") continue for ref in check["evidence_refs"]: @@ -467,7 +473,7 @@ def _claim(definition, protocol, prepared, record, actual, reader): ) refs.extend(used) conflict |= contradicted - if not proofs: + if not proofs or problems: missing.append(required["check_id"]) reasons.extend(problems) elif required["check_id"] == "cleanup" and not all(p["confirmed"] for p in proofs): diff --git a/tests/test_benchmark_assessment.py b/tests/test_benchmark_assessment.py index 83fdf210..2a1e8638 100644 --- a/tests/test_benchmark_assessment.py +++ b/tests/test_benchmark_assessment.py @@ -202,6 +202,63 @@ def open(self, ref): assert result["claims"][0]["missing_evidence"] == ["format"] +@pytest.mark.parametrize("reverse", [False, True], ids=["valid-first", "gap-first"]) +@pytest.mark.parametrize("kind,separate_check", [ + (kind, separate) for kind in ("UnsafeArtifactPath", "ArtifactUnavailable", + "ArtifactIntegrityError", "bad_bytes", "invalid_json") + for separate in (False, True) +] + [("empty", True)]) +def test_good_proof_cannot_hide_another_declared_evidence_gap( + retained, separate_check, reverse, kind, +): + good = retained["record"]["checks"][0] + good_refs = deepcopy(good["evidence_refs"]) + other, _ = retained["add"]("format", "failed") + target = other["evidence_refs"][0] + if kind == "invalid_json": + target = retained["put"]("invalid-proof.json", b"{}") + other["evidence_refs"] = [target] + if kind == "empty": + other["evidence_refs"] = [] + if not separate_check: + good["evidence_refs"].extend(other["evidence_refs"]) + retained["record"]["checks"].remove(other) + if reverse: + good["evidence_refs"].reverse() + elif reverse: + retained["record"]["checks"].reverse() + reader = ArtifactReader(retained["config"]) + + class Fault: + def open(self, ref): + if ref == target: + if kind in {"UnsafeArtifactPath", "ArtifactUnavailable", "ArtifactIntegrityError"}: + raise PublicError(kind, stage="open", message="fixture rejection") + if kind == "bad_bytes": + return BytesIO(b"not the retained failed proof") + return reader.open(ref) + + result = run(retained, reader=Fault()) + assert [c["verdict"] for c in result["claims"]] == ["INCONCLUSIVE", "PASS", "PASS"] + assert result["claims"][0]["missing_evidence"] == ["format"] + assert result["claims"][0]["evidence"] == good_refs + assert result["eligibility"][0]["status"] == "unknown" + + +@pytest.mark.parametrize("field,value", [ + ("record_ref", {}), ("claims", [{}]), ("comparison", [None]), + ("candidate", {"context_refs": None}), ("metrics", None), +]) +def test_public_identity_reports_malformed_nested_projection_as_declared_error( + retained, field, value, +): + result = run(retained) + result[field] = value + with pytest.raises(assessment.AssessmentError) as error: + assessment.assessment_id(result) + assert error.value.code == "InvalidRequest" + + @pytest.mark.parametrize("error", [TypeError("broken host adapter"), CanonicalError("host_adapter", "broken host adapter")], ids=["type", "canonical"])