diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index ebaf1b65..99852a52 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -37,7 +37,8 @@ "tests/test_apps_publication.py", "tests/test_apps_replay.py", "tests/test_benchmark_preparation.py", - "tests/test_result_delivery.py" + "tests/test_result_delivery.py", + "tests/test_benchmark_assessment.py" ] }, { @@ -60,7 +61,8 @@ "tests/test_apps_artifact_reader.py", "tests/test_apps_assess.py", "tests/test_apps_generation_plan.py", - "tests/test_apps_module_migrations.py" + "tests/test_apps_module_migrations.py", + "tests/test_benchmark_assessment.py" ] }, { @@ -139,7 +141,8 @@ "tests/test_apps_execution_binding.py", "tests/test_apps_generation_plan.py", "tests/test_apps_module_migrations.py", - "tests/test_benchmark_preparation.py" + "tests/test_benchmark_preparation.py", + "tests/test_benchmark_assessment.py" ] }, { diff --git a/docs/contracts/production-assess.md b/docs/contracts/production-assess.md new file mode 100644 index 00000000..a28c26b0 --- /dev/null +++ b/docs/contracts/production-assess.md @@ -0,0 +1,93 @@ +# Production P1 offline assessment + +`inferrsi.benchmark.assessment` implements the accepted D023 / `t108-v4` +family: `p1-fixture@1`, `p1-rules@1`, operation `evaluate`, scope `p1`. +It consumes the same frozen preparation as +[production prepare](production-prepare.md). The public ports are +`assess(prepared, evidence, *, artifacts)`, `encode`, `decode`, and +`validate_assessment`. `AssessmentService(clock=...)` supports reproducible +assessment timestamps; its default clock is real UTC. + +```python +from inferrsi.apps.api import assess as apps_assess +from inferrsi.benchmark.assessment import assess, encode, validate_assessment + +output = apps_assess( + saved_assess_request, workspace=workspace, storage_config=storage_config, + assess_benchmark=assess, encode_assessment=encode, + validate_assessment=validate_assessment, +) +``` + +Apps reads one fixed record and its preparation, checks the primary key against +the frozen protocol, and supplies its read-only artifact capability. Benchmark +does not import Apps, Harness or Knowledge, register a worker, execute a process, +use a network/device, archive results, or rewrite execution facts. Its encoder, +decoder and result validator perform no artifact reads. + +## Binding and byte authority + +The port snapshots the JSON inputs and checks the complete canonical record +against its RecordRef hash, size, attempt and revision. It checks task, +operation, scope, semantic `prepared_id` and `protocol_digest`. It does not +compare archived reference locations as identities. The actual frozen protocol +bytes are read and validated with the prepare port's pure validator; unsupported +task, grader or claim-rule revisions are explicitly refused. + +Every opened artifact is independently checked against its declared size and +SHA-256. The capability may supply bytes or a binary stream; assessment closes +streams it opens. Portable `UnsafeArtifactPath`, `ArtifactUnavailable` and +`ArtifactIntegrityError` exceptions are recognized by their exact `code`, without +depending on another module's exception class. An affected proof becomes a +claim-level evidence gap. Unknown host programming exceptions propagate. +Every declared proof for a matching check must be usable: a good proof does not +hide an unreadable, corrupt or invalid second proof, or another matching check +with no evidence. That claim remains INCONCLUSIVE, preserving the verified refs +and the gap reason, regardless of proof/check order. Other claims are independent. +Failure to obtain the protocol itself is a port error: missing rules cannot be +replaced by a guessed assessment. + +An input's actual hash is accepted only with matching retained bytes and a fixed +capture fact. Proof-declared candidate IDs/hashes do not authenticate themselves. +Check identity, revision, subject, phase, attempt, operation, scope and artifact +schema must match a frozen requirement. Producer kind/domain must be allowed; +the domain must match the actual target binding. Worker and sample evidence must +also match the observed runtime entry/revision/code pin and a started step bound +to that input. Caller assertions cannot substitute for those facts. + +## Frozen P1 decisions + +- F and C consume their own trustworthy format/grader proofs. A complete passed + proof yields PASS, a matching failed proof yields FAIL, and missing/invalid + evidence yields INCONCLUSIVE. Another claim's missing evidence does not erase + an independently supported result. +- M validates four positive Decimal samples in A1/B1/B2/A2 order with their + frozen round, role and `ms` unit. It also needs a completed successful process, + known closed steps, confirmed process cleanup, confirmed acquired-lease + releases and a matching cleanup proof. Missing terminal facts remain + INCONCLUSIVE; there is no independent-error exception for this terminal claim. +- Contradictory trustworthy proofs are aggregated into one INCONCLUSIVE claim + with a conflict reason, including conflicting sample values. No first/last + winner or duplicate claim key is emitted. P1 eligibility is ineligible for a + failed claim/conflict, unknown for incomplete evidence, and eligible only when + all three frozen conditions pass. It is not an overall verdict. +- This protocol freezes no metric definitions or assessment comparison target, + so metrics/comparison arrays remain empty. M is measurement validity, not a + claim of a service gain. + +Raw proof bytes retain their original hash. Their JSON need not use the DTO's +canonical whitespace: parsing still rejects duplicate keys, floating/nonfinite +numeric tokens, invalid Unicode and invalid P1 fields. New EvaluationResult +bytes follow `inferrsi-json-v1`. Its semantic ID excludes only the declared +clock/self-ID fields and schema-typed ArtifactRef locations, including nested +candidate references. All other supporting content participates. + +## Acceptance boundary + +The Apps primary-key order, missing-result-key error and archive-failure behavior +remain Apps responsibilities. Cross-tests use real production prepare/assess +ports, Apps archive/read/publish code and retained software-test record/proof +fixtures. They cover T35/T39/V21–V23, including isolated-process identity and +fixed-clock bytes. They do not claim that the production Harness evaluation +worker has run. That target and T37/T40 belong to #429; joint production CPU +acceptance remains necessary before #371 can be completed. diff --git a/docs/contracts/production-proofs.md b/docs/contracts/production-proofs.md new file mode 100644 index 00000000..3a22a07f --- /dev/null +++ b/docs/contracts/production-proofs.md @@ -0,0 +1,79 @@ +# P1 worker-report codecs and capture drafts + +`inferrsi.benchmark.proofs` owns pure `encode_report`, `decode_report`, +`validate_report`, `project`, and `ProofProjector` ports. This is the Benchmark +codec/projector slice of the accepted capture boundary in +[#565](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7063820381). +It does not implement a worker, grader predicate, baseline workload, sampler, +Harness capture lifecycle or Apps adapter. Their actual behavior and resource +pins remain prerequisites for shared production acceptance. + +## Closed raw report + +The frozen `fixture-grader@1` output contains exactly `schema_version: 1`, +`grader` and `samples`. The latter two are required and may be null for an +incomplete family. A non-null grader contains exactly `status` (`passed` or +`failed`) and `reasons` (portable identifiers). Samples contain the four frozen +identities A1/B1/B2/A2, baseline/candidate/candidate/baseline and rounds 1/1/2/2. +Each value is a positive canonical Decimal string with unit `ms`. + +No identity, Ref, producer, control domain or cleanup assertion is accepted from +the worker. JSON whitespace is allowed; duplicate keys, floating/nonfinite +numeric tokens, invalid Unicode and unsupported versions are rejected. Encoders +use the existing canonical v1 codec. Neither encoding nor validation opens files. + +## Frozen configuration and runtime inputs + +Construct `ProofProjector(prepared, protocol_bytes)` before attempting execution. +It snapshots the preparation and checks the existing task/grader/recipe/rule +support and complete protocol identity using the production preparation port. +Unsupported configuration raises `ProofError` with the existing portable code. + +Call `project(phase, facts, *, candidate_bytes, report_bytes)` on that object, or +use the one-shot function with `prepared` and `protocol_bytes` first. The two +phases are `preparing` and `finalizing`. Bytes are supplied by a host adapter; +this module receives no filesystem, network, process, archive or store handle. +`None` means the corresponding byte input is unavailable. An unexpected host +content type raises `TypeError` rather than becoming an evidence gap. + +`CONTEXT_FIELDS` specifies a closed version-1 wire projection of existing H +facts: request/run/attempt/digest, preparation/protocol/operation/scope/candidate, +actual bound inputs and target/runtime binding, steps, started/outcome/exit code, +cleanup and leases. Unavailable terminal fields are null. The host supplies a +defensive snapshot; no shared Python business DTO is introduced. + +The candidate bytes must match the actual retained input's digest, size, attempt +and capture fact. Any frozen candidate expectation also applies. Worker report +families require an actually started input-bound step with the observed entry, +revision and code digest matching the frozen recipe. Allowed control domains +must match the actual target; changing a locator does not change identity. +These checks do not authenticate a caller-provided context: H owns the context, +read capability and final producer sealing. + +## Draft output and refusal + +The return is `(draft_wire, proof_blobs)`. Draft wire has exactly +`schema_version`, `checks` and `errors`. Each check has the frozen check identity, +revision, subject, phase and artifact schema, plus status, reasons, `proof_key` +and source. Sources are `candidate_capture`, `worker_report`, `cleanup_capture`. +`proof_key` is null when no proof was produced. The separate map associates keys +with immutable canonical proof bytes. Errors contain a portable `code`, affected +`check_ids`, and an identifier `reason`. + +The preparing phase checks actual UTF-8 bytes and drafts format-proof@1. Invalid +UTF-8 gives a failed format proof, not a host error or an invented correctness +failure. This projector does not decide whether to start a worker. + +The finalizing phase projects completed grader/sample families into the existing +grader-proof@1 and samples-proof@1. Attempt/candidate/hash fields come from H +facts, never report declarations. A missing or invalid family is not_performed +without a proof. A valid grader result survives invalid sample semantics, and +vice versa; invalid raw JSON or a foreign report envelope invalidates the report +itself. The cleanup proof comes only from observed cleanup and acquired-lease +release facts. Unknown or failed cleanup never becomes confirmed. + +H must validate drafts, retain raw and derived bytes before its record commit, +compute Refs itself and seal actual producer/domain/route/attempt fields. A draft +is neither a trustworthy CheckObservation nor a committed Ref. Existing offline +assessment and V21/V22 rules remain unchanged. This slice alone cannot make a +real execution record contain matching proofs or establish complete F/C/M. diff --git a/docs/operations.md b/docs/operations.md index 9115ca36..743c4cd2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -53,6 +53,8 @@ Effect classes used below: | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | +| Assess one fixed P1 record offline | Python `benchmark.assessment.assess`; [production assess](contracts/production-assess.md) | Read-only retained evidence / pure encoding | Verifies the frozen protocol, actual capture and proof bytes; returns independent claims without launching a worker or writing an archive; Apps selects the primary key and archives the result | +| Project P1 worker reports into capture drafts | Python `benchmark.proofs` codecs / `ProofProjector.project`; [proof drafts](contracts/production-proofs.md) | Pure computation | Produces proof bytes from supplied capture facts and reports; H must authenticate, retain and seal them before commit; no worker or grader execution | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | | Retain an executed retrieval | Python `apps.workflows.campaign.save_retrieval`; [capture contract](knowledge-retrieval-trace.md#retain-a-standalone-result) | Local evidence-writing | Saves an existing catalog/collect result under a fresh campaign capture ID; no new search or automatic publication | diff --git a/src/inferrsi/benchmark/assessment.py b/src/inferrsi/benchmark/assessment.py new file mode 100644 index 00000000..c04e376b --- /dev/null +++ b/src/inferrsi/benchmark/assessment.py @@ -0,0 +1,574 @@ +"""Read-only D023/P1 assessment of one fixed record and its retained evidence.""" + +from __future__ import annotations + +import json +import re +from contextlib import closing +from copy import deepcopy +from datetime import datetime, timezone +from decimal import Decimal +from hashlib import sha256 + +from ..contracts.json_v1 import CanonicalError +from ..contracts.json_v1 import decode as canonical_decode +from ..contracts.json_v1 import encode as canonical_encode +from . import preparation + +_RESULT = set("schema_version assessment_id task grader assessor prepared_ref record_ref " + "protocol_digest candidate attempt_id claims metrics eligibility comparison " + "evidence_artifacts assessed_at supersedes derived_from limitations".split()) +_CLAIM = set("claim_id validation_scope verdict reasons evidence " + "missing_evidence limitations".split()) +_RECORD = set("schema_version request_id run_id attempt_id request_digest owner target_request " + "target_binding retry_of retry_reason generation_record_ref prepared_ref " + "protocol_digest operation validation_scope candidate record_version recorded_at " + "bound_inputs phase started side_effect_unknown process_outcome exit_code " + "elapsed_seconds steps cancellation cleanup artifacts checks budget_admission " + "usage resource_leases errors".split()) +_COMMON = {"schema_version", "candidate_id", "actual_sha256", "attempt_id", "operation", + "validation_scope"} +_ITEM_ERRORS = {"UnsafeArtifactPath", "ArtifactUnavailable", "ArtifactIntegrityError"} + + +class AssessmentError(ValueError): + """A declared Benchmark rejection, independent of any Harness exception class.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + def to_json(self) -> dict: + return { + "code": self.code, + "stage": "assessment_binding" if self.code == "IdentityMismatch" else "assess", + "attempt_id": None, "started": None, "latest_record_ref": None, + "partial_record": None, "retained_refs": [], "field_errors": [], + "message": str(self), + } + + +def _require(condition, message="Invalid P1 assessment.", code="InvalidRequest"): + if not condition: + raise AssessmentError(code, message) + + +def _object(value, fields): + _require(isinstance(value, dict) and value.keys() == fields) + + +def _artifact(value): + preparation._artifact(value) + _require(all(ord(char) >= 32 and ord(char) != 127 for char in value["location"]), + "Artifact paths must not contain control characters.") + + +def _strings(value, *, identifiers=False): + _require(isinstance(value, list) and all( + preparation._id(item) if identifiers else isinstance(item, str) for item in value + )) + + +def _timestamp(value): + _require(isinstance(value, str) and re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z", value + ) is not None) + try: + datetime.fromisoformat(value[:19]) + except ValueError: + raise AssessmentError("InvalidRequest", "Invalid assessment timestamp.") from None + + +def _projection(result): + """Project only schema-declared ArtifactRefs, including nested candidate refs.""" + # JSON values have no alias identity: callers may reuse the same dict at + # several typed paths without making the projection depend on object aliasing. + value = canonical_decode(canonical_encode(result)) + del value["assessment_id"], value["assessed_at"] + refs = [value["prepared_ref"], value["record_ref"], *value["derived_from"]] + refs.extend(item["against"] for item in value["comparison"]) + if value["supersedes"] is not None: + refs.append(value["supersedes"]) + artifacts = [ref["artifact"] for ref in refs] + artifacts.extend(value["evidence_artifacts"]) + for items, field in ((value["claims"], "evidence"), (value["metrics"], "raw_samples")): + artifacts.extend(ref for item in items for ref in item[field]) + if value["candidate"] is not None: + artifacts.extend(value["candidate"]["context_refs"]) + for ref in artifacts: + _artifact(ref) + del ref["location"] + return value + + +def assessment_id(result: dict) -> str: + """Pure v1 identity; only assessed_at, the self ID and typed locations are excluded.""" + _object(result, _RESULT) + try: + return "sha256:" + sha256(canonical_encode(_projection(result))).hexdigest() + except (KeyError, TypeError, CanonicalError, preparation.PreparationError): + raise AssessmentError( + "InvalidRequest", "Malformed assessment identity projection.", + ) from None + + +def validate_assessment(result: dict) -> None: + """Pure validation of the supported P1 result and its semantic identity.""" + canonical_encode(result) + _object(result, _RESULT) + _require(type(result["schema_version"]) is int and result["schema_version"] == 1, + code="UnsupportedSchema") + _require(result["task"] == {"name": "p1-fixture", "revision": "1"} + and result["grader"] == {"name": "p1-rules", "revision": "1"} + and result["assessor"] == {"name": "fixture-assessor", "revision": "1"}, + code="UnsupportedRuleVersion") + _timestamp(result["assessed_at"]) + _require(preparation._id(result["attempt_id"])) + _require(isinstance(result["protocol_digest"], str) + and re.fullmatch(r"sha256:[a-f0-9]{64}", result["protocol_digest"]) is not None) + for field, fields in (("prepared_ref", {"schema_version", "prepared_id", "artifact"}), + ("record_ref", {"attempt_id", "record_version", "artifact"})): + _object(result[field], fields) + _artifact(result[field]["artifact"]) + _require(result["prepared_ref"]["schema_version"] == 1 + and type(result["prepared_ref"]["schema_version"]) is int + and re.fullmatch(r"sha256:[a-f0-9]{64}", + result["prepared_ref"]["prepared_id"]) is not None) + _require(result["record_ref"]["attempt_id"] == result["attempt_id"] + and type(result["record_ref"]["record_version"]) is int + and result["record_ref"]["record_version"] > 0) + candidate = result["candidate"] + if candidate is not None: + _object(candidate, {"candidate_id", "kind", "input_role", "parent_id", + "parent_digest", "context_refs"}) + _require(all(preparation._id(candidate[key]) + for key in ("candidate_id", "kind", "input_role"))) + _require(isinstance(candidate["context_refs"], list)) + for ref in candidate["context_refs"]: + _artifact(ref) + _require(candidate["parent_id"] is None or preparation._id(candidate["parent_id"])) + _require(candidate["parent_digest"] is None or isinstance(candidate["parent_digest"], str) + and preparation._SHA.fullmatch(candidate["parent_digest"]) is not None) + _require(isinstance(result["claims"], list) and len(result["claims"]) == 3) + keys = [] + for claim in result["claims"]: + _object(claim, _CLAIM) + keys.append((claim["claim_id"], claim["validation_scope"])) + _require(claim["verdict"] in {"PASS", "FAIL", "INCONCLUSIVE"}) + for field in ("reasons", "missing_evidence"): + _strings(claim[field], identifiers=True) + _strings(claim["limitations"]) + _require(isinstance(claim["evidence"], list)) + for ref in claim["evidence"]: + _artifact(ref) + _require(len(set(keys)) == len(keys) and set(keys) == {("F", "p1"), ("C", "p1"), ("M", "p1")}) + # P1 freezes no metric definitions or assessment comparison target. + _require(result["metrics"] == [] and result["comparison"] == [] + and result["supersedes"] is None and result["derived_from"] == []) + _require(isinstance(result["eligibility"], list) and len(result["eligibility"]) == 1) + eligibility = result["eligibility"][0] + _object(eligibility, {"validation_scope", "status", "reasons"}) + _require(eligibility["validation_scope"] == "p1" + and eligibility["status"] in {"eligible", "ineligible", "unknown"}) + _strings(eligibility["reasons"], identifiers=True) + _strings(result["limitations"]) + _require(isinstance(result["evidence_artifacts"], list)) + for ref in result["evidence_artifacts"]: + _artifact(ref) + _require(result["assessment_id"] == assessment_id(result), "Invalid assessment identity.", + "IdentityMismatch") + + +def encode(result: dict) -> bytes: + validate_assessment(result) + return canonical_encode(result) + + +def decode(data: bytes) -> dict: + result = canonical_decode(data) + _require(canonical_encode(result) == data, "Assessment bytes are not canonical.") + validate_assessment(result) + return result + + +class _RetainedBytes: + def __init__(self, artifacts): + self.artifacts = artifacts + self.cache = {} + self.verified = [] + + def read(self, ref): + try: + _artifact(ref) + except (preparation.PreparationError, AssessmentError): + return None, "invalid_artifact_ref" + key = canonical_encode(ref) + if key in self.cache: + return self.cache[key] + try: + source = self.artifacts.open(deepcopy(ref)) + if isinstance(source, bytes): + data = source + else: + chunks, remaining = [], ref["size_bytes"] + 1 + with closing(source): + while remaining > 0: + chunk = source.read(min(remaining, 1024 * 1024)) + if not isinstance(chunk, bytes): + raise TypeError("ArtifactReader must supply binary bytes or streams.") + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + except Exception as error: + # Portable error codes are the capability contract; importing an Apps + # or Harness exception class would invert the module dependency. + if getattr(error, "code", None) not in _ITEM_ERRORS: + raise + self.cache[key] = (None, error.code) + return self.cache[key] + if len(data) != ref["size_bytes"] or sha256(data).hexdigest() != ref["sha256"]: + self.cache[key] = (None, "ArtifactIntegrityError") + else: + self.cache[key] = (data, None) + self.verified.append(deepcopy(ref)) + return self.cache[key] + + +def _binding(prepared, evidence): + preparation.encode(prepared) + _artifact(prepared["protocol"]) + for resource in prepared["resources"]: + _artifact(resource["artifact"]) + _object(evidence, {"schema_version", "record_ref", "record_digest", "task", "task_source", + "record", "artifact_checks"}) + record, ref = evidence["record"], evidence["record_ref"] + _object(record, _RECORD) + _object(ref, {"attempt_id", "record_version", "artifact"}) + _object(record["prepared_ref"], {"schema_version", "prepared_id", "artifact"}) + _artifact(ref["artifact"]) + _require(type(evidence["schema_version"]) is int and evidence["schema_version"] == 1 + and type(record["schema_version"]) is int and record["schema_version"] == 1) + data = canonical_encode(record) + _require(evidence["task"] == prepared["task"] and evidence["task_source"] == "prepared_ref" + and evidence["record_digest"] == ref["artifact"]["sha256"] + and (sha256(data).hexdigest(), len(data)) + == (ref["artifact"]["sha256"], ref["artifact"]["size_bytes"]) + and record["attempt_id"] == ref["attempt_id"] + and record["record_version"] == ref["record_version"] + and record["prepared_ref"]["prepared_id"] == prepared["prepared_id"] + and all(record[key] == prepared[key] + for key in ("protocol_digest", "operation", "validation_scope")), + "Prepared and fixed record identities differ.", "IdentityMismatch") + for field in ("checks", "bound_inputs", "steps", "artifact_checks"): + _require(isinstance((evidence if field == "artifact_checks" else record)[field], list)) + if record["candidate"] is not None: + _require(isinstance(record["candidate"], dict) + and all(record["candidate"].get(key) == prepared["candidate_contract"][key] + for key in ("kind", "input_role")), + "Candidate contract differs from the frozen preparation.", "IdentityMismatch") + if prepared["expected_candidate"] is not None: + _require(record["candidate"] is not None + and record["candidate"]["candidate_id"] + == prepared["expected_candidate"]["candidate_id"], + "Declared candidate identity differs.", "IdentityMismatch") + for item in record["bound_inputs"]: + if (isinstance(item, dict) and item.get("role") == record["candidate"]["input_role"] + and item.get("attempt_id") == record["attempt_id"] + and item.get("expected_sha256") is not None): + _require(item["expected_sha256"] == prepared["expected_candidate"]["sha256"], + "Declared candidate expectation differs.", "IdentityMismatch") + return record + + +def _captured_candidate(prepared, record, reader): + candidate = record["candidate"] + if not isinstance(candidate, dict): + return None + matches = [item for item in record["bound_inputs"] if isinstance(item, dict) + and item.get("role") == candidate.get("input_role") + and item.get("attempt_id") == record["attempt_id"]] + if len(matches) != 1 or candidate.get("kind") != prepared["candidate_contract"]["kind"]: + return None + item = matches[0] + ref, digest = item.get("retained_ref"), item.get("actual_sha256") + if not isinstance(ref, dict) or digest != ref.get("sha256") or item.get("captured_at") is None: + return None + try: + _timestamp(item["captured_at"]) + except AssessmentError: + return None + data, _ = reader.read(ref) + return digest if data is not None else None + + +def _trusted(check, required, prepared, record): + fields = {"check_id": required["check_id"], "checker_revision": required["checker_revision"], + "subject": required["subject"], "phase": required["phase"], + "artifact_schema": required["artifact_schema"], "operation": record["operation"], + "validation_scope": record["validation_scope"], "attempt_id": record["attempt_id"]} + if any(check.get(key) != value for key, value in fields.items()): + return False + producer, binding = check.get("producer"), record["target_binding"] + if not isinstance(producer, dict) or not isinstance(binding, dict): + return False + if (producer.get("kind") not in required["allowed_producer_kinds"] + or producer.get("control_domain") not in required["allowed_control_domains"] + or producer.get("control_domain") != binding.get("control_domain") + or not preparation._id(producer.get("capture_route"))): + return False + worker = producer["kind"] == "task_worker" + if worker or required["check_id"] == "samples": + runtimes = binding.get("runtime_bindings_by_step") + if (record["started"] is not True or not isinstance(record["candidate"], dict) + or not isinstance(runtimes, list)): + return False + recipe = prepared["recipe"] + step_ids = {step["step_id"] for step in record["steps"] if isinstance(step, dict) + and preparation._id(step.get("step_id")) + and step.get("status") != "not_started" + and (step.get("started_at") is not None or step.get("status") + in {"running", "exited", "timed_out", "cancelled"})} + input_steps = {step for item in record["bound_inputs"] if isinstance(item, dict) + and item.get("role") == record["candidate"]["input_role"] + and item.get("attempt_id") == record["attempt_id"] + and isinstance(item.get("step_ids"), list) + for step in item["step_ids"] if preparation._id(step)} + if not any(isinstance(runtime, dict) + and runtime.get("step_id") in step_ids & input_steps + and runtime.get("runtime_role") == recipe["runtime_role"] + and runtime.get("entrypoint") == recipe["entrypoint"] + and runtime.get("entrypoint_revision") == recipe["entrypoint_revision"] + and runtime.get("code_sha256") == recipe["code_sha256"] + for runtime in runtimes): + return False + if worker: + return (producer.get("identity") == recipe["entrypoint"] + and producer.get("revision") == recipe["entrypoint_revision"]) + return preparation._id(producer.get("identity")) + + +def _proof_json(data): + # Raw evidence is hashed as retained, not rewritten into the DTO encoding. + # JSON whitespace is allowed; duplicate keys, non-finite/floating tokens, + # invalid Unicode and out-of-range integers must still be refused. + def pairs(items): + result = {} + for key, item in items: + _require(key not in result, "Duplicate evidence key.") + result[key] = item + return result + + def number(_): + raise AssessmentError("InvalidRequest", "Evidence requires Decimal strings.") + + try: + value = json.loads(data.decode("utf-8"), object_pairs_hook=pairs, + parse_float=number, parse_constant=number) + canonical_encode(value) + except (ValueError, UnicodeError, RecursionError): + raise AssessmentError("InvalidRequest", "Invalid P1 evidence JSON.") from None + return value + + +def _proof(data, required, record, actual): + value = _proof_json(data) + key = required["check_id"] + common = {"schema_version": 1, "attempt_id": record["attempt_id"], + "operation": record["operation"], "validation_scope": record["validation_scope"]} + if key == "cleanup": + _object(value, set(common) | {"confirmed"}) + _require(type(value["confirmed"]) is bool) + else: + common.update(candidate_id=record["candidate"]["candidate_id"], actual_sha256=actual) + _require(actual is not None) + if key in ("format", "grader"): + _object(value, _COMMON | {"claim_id", "checker_revision", "status", "reasons"}) + _require(value["claim_id"] == required["for_claims"][0] + and value["checker_revision"] == required["checker_revision"] + and value["status"] in {"passed", "failed"}) + _strings(value["reasons"], identifiers=True) + else: + _object(value, _COMMON | {"samples"}) + _require(isinstance(value["samples"], list) and len(value["samples"]) == 4) + for sample, identity in zip(value["samples"], ( + ("A1", "baseline", 1), ("B1", "candidate", 1), + ("B2", "candidate", 2), ("A2", "baseline", 2), + )): + _object(sample, {"sample_id", "role", "round", "value", "unit"}) + _require((sample["sample_id"], sample["role"], sample["round"]) == identity + and type(sample["round"]) is int and sample["unit"] == "ms" + and isinstance(sample["value"], str) + and preparation._DECIMAL.fullmatch(sample["value"]) is not None + and Decimal(sample["value"]) > 0) + _require(type(value["schema_version"]) is int + and all(value[field] == expected for field, expected in common.items())) + return value + + +def _observations(required, prepared, record, actual, reader): + proofs, refs, problems = [], [], [] + for check in record["checks"]: + if not isinstance(check, dict) or check.get("check_id") != required["check_id"]: + continue + if (not _trusted(check, required, prepared, record) + or check.get("status") not in {"passed", "failed"} + or not isinstance(check.get("evidence_refs"), list) + or not check["evidence_refs"]): + problems.append("untrusted_or_incomplete_check") + continue + for ref in check["evidence_refs"]: + data, reason = reader.read(ref) + if data is None: + problems.append(reason) + continue + try: + proof = _proof(data, required, record, actual) + if "status" in proof: + _require(proof["status"] == check["status"]) + except (CanonicalError, AssessmentError, KeyError, TypeError): + problems.append("invalid_evidence") + continue + proofs.append(proof) + refs.append(ref) + key = required["check_id"] + signatures = {canonical_encode(proof if key == "samples" else proof.get( + "status", proof.get("confirmed"))) for proof in proofs} + return proofs, refs, problems, len(signatures) > 1 + + +def _unique(refs): + return list({canonical_encode(ref): deepcopy(ref) for ref in refs}.values()) + + +def _cleanup_confirmed(record): + cleanup, resources = record["cleanup"], record["resource_leases"] + if (not isinstance(cleanup, dict) or not isinstance(resources, dict) + or not isinstance(resources.get("leases"), list)): + return False + if (cleanup.get("status") != "confirmed" or cleanup.get("process_exit_confirmed") is not True + or resources.get("applicability") not in {"applicable", "not_applicable"} + or resources.get("applicability") == "not_applicable" and resources["leases"] + or resources.get("applicability") == "applicable" and not resources["leases"]): + return False + for lease in resources["leases"]: + if not isinstance(lease, dict) or lease.get("release_confirmed") is not True: + return False + try: + _timestamp(lease.get("release_confirmed_at")) + except AssessmentError: + return False + return True + + +def _claim(definition, protocol, prepared, record, actual, reader): + missing, reasons, refs, decisions = [], [], [], [] + conflict = False + for required in protocol["required_evidence"]: + if definition["claim_id"] not in required["for_claims"]: + continue + proofs, used, problems, contradicted = _observations( + required, prepared, record, actual, reader, + ) + refs.extend(used) + conflict |= contradicted + if not proofs or problems: + missing.append(required["check_id"]) + reasons.extend(problems) + elif required["check_id"] == "cleanup" and not all(p["confirmed"] for p in proofs): + missing.append("cleanup") + else: + decisions.extend(p.get("status", "passed") for p in proofs) + reasons.extend(reason for p in proofs for reason in p.get("reasons", [])) + if definition["requires_completed_measurement"] and not ( + record["phase"] == "finished" and record["started"] is True + and record["process_outcome"] == "exited" and record["exit_code"] == 0 + and record["side_effect_unknown"] is False + and record["steps"] and all(isinstance(step, dict) and step.get("status") == "exited" + and step.get("exit_code") == 0 for step in record["steps"]) + ): + missing.append("completed_measurement") + if definition["requires_cleanup_confirmed"] and not _cleanup_confirmed(record): + missing.append("cleanup_confirmation") + if conflict: + reasons.append("conflicting_evidence") + if missing: + reasons.append("required_evidence_missing") + verdict = "INCONCLUSIVE" if conflict or missing else ( + "FAIL" if "failed" in decisions else "PASS" + ) + return { + "claim_id": definition["claim_id"], "validation_scope": definition["validation_scope"], + "verdict": verdict, "reasons": list(dict.fromkeys(reasons)), "evidence": _unique(refs), + "missing_evidence": list(dict.fromkeys(missing)), "limitations": [], + } + + +class AssessmentService: + """The offline P1 port; a clock is injected only for reproducible timestamps.""" + + def __init__(self, *, clock=None): + self._clock = clock or ( + lambda: datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + ) + + def assess(self, prepared: dict, evidence: dict, *, artifacts) -> dict: + try: + prepared = canonical_decode(canonical_encode(prepared)) + evidence = canonical_decode(canonical_encode(evidence)) + except CanonicalError: + return AssessmentError( + "InvalidRequest", "Malformed canonical assessment input.", + ).to_json() + try: + try: + record = _binding(prepared, evidence) + except (KeyError, TypeError): + raise AssessmentError("InvalidRequest", "Malformed fixed evidence input.") from None + reader = _RetainedBytes(artifacts) + data, reason = reader.read(prepared["protocol"]) + _require(data is not None, "Frozen protocol unavailable.", + reason if reason in _ITEM_ERRORS else "InvalidRequest") + try: + preparation.validate_prepared(prepared, data) + protocol = canonical_decode(data) + except CanonicalError: + raise AssessmentError("InvalidRequest", "Invalid frozen protocol bytes.") from None + for resource in preparation._closure(protocol): + _artifact(resource["artifact"]) + actual = _captured_candidate(prepared, record, reader) + claims = [_claim(item, protocol, prepared, record, actual, reader) + for item in protocol["claim_definitions"]] + verdicts = {claim["verdict"] for claim in claims} + conflict = any("conflicting_evidence" in claim["reasons"] for claim in claims) + status = "ineligible" if conflict or "FAIL" in verdicts else ( + "eligible" if verdicts == {"PASS"} else "unknown" + ) + result = { + "schema_version": 1, "assessment_id": "", "task": deepcopy(prepared["task"]), + "grader": deepcopy(prepared["grader"]), + "assessor": {"name": "fixture-assessor", "revision": "1"}, + "prepared_ref": deepcopy(record["prepared_ref"]), + "record_ref": deepcopy(evidence["record_ref"]), + "protocol_digest": record["protocol_digest"], + "candidate": deepcopy(record["candidate"]), "attempt_id": record["attempt_id"], + "claims": claims, "metrics": [], "comparison": [], + "eligibility": [{"validation_scope": "p1", "status": status, + "reasons": ["conflicting_evidence"] if conflict else ( + ["claim_failed"] if status == "ineligible" else ( + [] if status == "eligible" else ["incomplete_evidence"] + ))}], + "evidence_artifacts": reader.verified, "assessed_at": self._clock(), + "supersedes": None, "derived_from": [], "limitations": [], + } + result["assessment_id"] = assessment_id(result) + validate_assessment(result) + return result + except (AssessmentError, preparation.PreparationError) as error: + return AssessmentError(error.code, str(error)).to_json() + + +def assess(prepared: dict, evidence: dict, *, artifacts) -> dict: + """Assess the first frozen task using a real UTC assessment clock.""" + return AssessmentService().assess(prepared, evidence, artifacts=artifacts) diff --git a/src/inferrsi/benchmark/proofs.py b/src/inferrsi/benchmark/proofs.py new file mode 100644 index 00000000..a1913cb3 --- /dev/null +++ b/src/inferrsi/benchmark/proofs.py @@ -0,0 +1,300 @@ +"""Pure P1 worker-report codecs and drafts for a host-owned capture lifecycle. + +Drafts are not observations or committed references. Harness must authenticate +their source, retain the bytes and seal producer/attempt fields before commit. +This module neither runs a grader nor manufactures measurement samples. +""" + +from __future__ import annotations + +from decimal import Decimal +from hashlib import sha256 + +from ..contracts.json_v1 import CanonicalError +from ..contracts.json_v1 import decode as canonical_decode +from ..contracts.json_v1 import encode as canonical_encode +from . import assessment, preparation + +CONTEXT_FIELDS = frozenset( + "schema_version request_id run_id attempt_id request_digest prepared_ref protocol_digest " + "operation validation_scope candidate bound_inputs target_binding steps started " + "process_outcome exit_code cleanup resource_leases".split() +) +_ORDER = (("A1", "baseline", 1), ("B1", "candidate", 1), + ("B2", "candidate", 2), ("A2", "baseline", 2)) + + +class ProofError(ValueError): + """A declared codec/configuration rejection; no dependency on Harness types.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + +def _require(condition, message="Invalid proof projection input.", code="InvalidRequest"): + if not condition: + raise ProofError(code, message) + + +def _object(value, fields): + _require(isinstance(value, dict) and value.keys() == fields) + + +def _snapshot(value): + try: + return canonical_decode(canonical_encode(value)) + except CanonicalError: + raise ProofError("InvalidRequest", "Invalid canonical projection input.") from None + + +def _report_envelope(data): + if not isinstance(data, bytes): + raise TypeError("A worker report must be bytes.") + try: + value = assessment._proof_json(data) + except (CanonicalError, assessment.AssessmentError) as error: + raise ProofError("InvalidRequest", "Invalid worker report JSON.") from error + _object(value, {"schema_version", "grader", "samples"}) + _require(type(value["schema_version"]) is int and value["schema_version"] == 1, + "Unsupported worker report version.", "UnsupportedSchema") + return value + + +def _grader(value): + _object(value, {"status", "reasons"}) + _require(value["status"] in ("passed", "failed") + and isinstance(value["reasons"], list) + and all(preparation._id(reason) for reason in value["reasons"])) + + +def _samples(values): + _require(isinstance(values, list) and len(values) == len(_ORDER)) + for item, identity in zip(values, _ORDER): + _object(item, {"sample_id", "role", "round", "value", "unit"}) + _require(type(item["round"]) is int + and (item["sample_id"], item["role"], item["round"]) == identity + and item["unit"] == "ms" and isinstance(item["value"], str) + and preparation._DECIMAL.fullmatch(item["value"]) is not None + and Decimal(item["value"]) > 0) + + +def validate_report(value: dict) -> None: + """Validate the complete closed fixture-grader@1 report without I/O.""" + value = _report_envelope(canonical_encode(_snapshot(value))) + if value["grader"] is not None: + _grader(value["grader"]) + if value["samples"] is not None: + _samples(value["samples"]) + + +def encode_report(value: dict) -> bytes: + validate_report(value) + return canonical_encode(value) + + +def decode_report(data: bytes) -> dict: + """Raw report whitespace is allowed; strict v1 JSON and family rules apply.""" + value = _report_envelope(data) + validate_report(value) + return value + + +def _context(facts, prepared): + _object(facts, CONTEXT_FIELDS) + _require(type(facts["schema_version"]) is int and facts["schema_version"] == 1, + "Unsupported capture context.", "UnsupportedSchema") + _require(all(preparation._id(facts[key]) for key in ("request_id", "run_id", "attempt_id")) + and facts["request_id"] == facts["attempt_id"]) + digest = facts["request_digest"] + _require(isinstance(digest, str) and digest.startswith("sha256:") + and preparation._SHA.fullmatch(digest[7:]) is not None) + ref = facts["prepared_ref"] + _object(ref, {"schema_version", "prepared_id", "artifact"}) + try: + preparation._artifact(ref["artifact"]) + except preparation.PreparationError as error: + raise ProofError(error.code, str(error)) from error + _require(type(ref["schema_version"]) is int and ref["schema_version"] == 1 + and ref["prepared_id"] == prepared["prepared_id"] + and all(facts[key] == prepared[key] + for key in ("protocol_digest", "operation", "validation_scope")), + "Capture context differs from the preparation.", "IdentityMismatch") + candidate = facts["candidate"] + _require(isinstance(candidate, dict) and preparation._id(candidate.get("candidate_id")) + and all(candidate.get(key) == prepared["candidate_contract"][key] + for key in ("kind", "input_role")), + "Candidate contract differs.", "IdentityMismatch") + expected = prepared["expected_candidate"] + _require(expected is None or candidate["candidate_id"] == expected["candidate_id"], + "Candidate declaration differs.", "IdentityMismatch") + _require(isinstance(facts["bound_inputs"], list) and isinstance(facts["steps"], list) + and type(facts["started"]) is bool and isinstance(facts["target_binding"], dict)) + + +def _candidate_digest(facts, data, prepared): + if data is None: + return None + if not isinstance(data, bytes): + raise TypeError("Captured candidate content must be bytes or None.") + matches = [item for item in facts["bound_inputs"] if isinstance(item, dict) + and item.get("role") == facts["candidate"]["input_role"] + and item.get("attempt_id") == facts["attempt_id"]] + _require(len(matches) == 1, "Candidate capture is ambiguous.", "IdentityMismatch") + item = matches[0] + digest, ref = sha256(data).hexdigest(), item.get("retained_ref") + _require(isinstance(ref, dict) and item.get("actual_sha256") == ref.get("sha256") == digest + and type(ref.get("size_bytes")) is int and ref["size_bytes"] == len(data) + and item.get("captured_at") is not None and item.get("snapshot_format") == "file-v1", + "Candidate bytes differ from the actual capture.", "IdentityMismatch") + try: + preparation._artifact(ref) + assessment._timestamp(item["captured_at"]) + except (preparation.PreparationError, assessment.AssessmentError) as error: + raise ProofError("IdentityMismatch", "Invalid actual candidate capture.") from error + expected = prepared["expected_candidate"] + _require(expected is None or expected["sha256"] == digest, + "Actual candidate differs from its frozen expectation.", "IdentityMismatch") + return digest + + +def _worker_bound(facts, prepared): + if facts["started"] is not True: + return False + runtimes = facts["target_binding"].get("runtime_bindings_by_step") + if not isinstance(runtimes, list): + return False + started = {step.get("step_id") for step in facts["steps"] if isinstance(step, dict) + and step.get("status") != "not_started" and step.get("started_at") is not None + and preparation._id(step.get("step_id"))} + inputs = {step for item in facts["bound_inputs"] if isinstance(item, dict) + and item.get("role") == facts["candidate"]["input_role"] + and item.get("attempt_id") == facts["attempt_id"] + and isinstance(item.get("step_ids"), list) + for step in item["step_ids"] if preparation._id(step)} + recipe = prepared["recipe"] + return any(isinstance(runtime, dict) and preparation._id(runtime.get("step_id")) + and runtime.get("step_id") in started & inputs + and all(runtime.get(key) == recipe[key] for key in + ("runtime_role", "entrypoint", "entrypoint_revision", "code_sha256")) + for runtime in runtimes) + + +class ProofProjector: + """Freeze supported configuration before execution; project supplied bytes only.""" + + def __init__(self, prepared: dict, protocol_bytes: bytes): + self._prepared = _snapshot(prepared) + try: + preparation.validate_prepared(self._prepared, protocol_bytes) + except (CanonicalError, preparation.PreparationError) as error: + raise ProofError(getattr(error, "code", "InvalidRequest"), str(error)) from error + + def project(self, phase, facts, *, candidate_bytes, report_bytes): + """Return (draft wire, proof bytes), never producer identities or valid Refs.""" + _require(phase in ("preparing", "finalizing"), "Unsupported capture phase.") + if report_bytes is not None and not isinstance(report_bytes, bytes): + raise TypeError("A worker report must be bytes or None.") + prepared, facts = self._prepared, _snapshot(facts) + _context(facts, prepared) + if phase == "preparing": + _require(facts["started"] is False + and all(facts[key] is None for key in + ("process_outcome", "exit_code", "cleanup", "resource_leases")), + "Preparing capture must precede worker and terminal observations.", + "IdentityMismatch") + actual = _candidate_digest(facts, candidate_bytes, prepared) + required = {item["check_id"]: item for item in prepared["required_evidence"]} + draft, blobs = {"schema_version": 1, "checks": [], "errors": []}, {} + common = {"schema_version": 1, "attempt_id": facts["attempt_id"], + "operation": facts["operation"], "validation_scope": facts["validation_scope"]} + candidate = {**common, "candidate_id": facts["candidate"]["candidate_id"], + "actual_sha256": actual} + + def emit(key, source, status, proof=None, reasons=(), error=None): + rule = required[key] + allowed = (facts["target_binding"].get("control_domain") + in rule["allowed_control_domains"]) + if not allowed: + status, proof, reasons, error = ( + "not_performed", None, ["control_domain_mismatch"], "IdentityMismatch", + ) + proof_key = key if proof is not None else None + draft["checks"].append({ + **{field: rule[field] for field in + ("check_id", "checker_revision", "subject", "phase", "artifact_schema")}, + "status": status, "reasons": list(reasons), "proof_key": proof_key, + "source": source, + }) + if proof is not None: + blobs[key] = canonical_encode(proof) + if error: + draft["errors"].append({"code": error, "check_ids": [key], + "reason": reasons[0]}) + + if phase == "preparing": + if actual is None: + emit("format", "candidate_capture", "not_performed", + reasons=["candidate_bytes_unavailable"], error="ArtifactUnavailable") + else: + try: + candidate_bytes.decode("utf-8") + status, reasons = "passed", [] + except UnicodeError: + status, reasons = "failed", ["invalid_utf8"] + emit("format", "candidate_capture", status, { + **candidate, "claim_id": "F", "checker_revision": "1", + "status": status, "reasons": reasons, + }, reasons) + return _snapshot(draft), dict(blobs) + + report, report_error, report_reason = None, None, None + if report_bytes is None: + report_error, report_reason = "ArtifactUnavailable", "worker_report_unavailable" + elif actual is None: + report_error, report_reason = "ArtifactUnavailable", "candidate_bytes_unavailable" + elif not _worker_bound(facts, prepared): + report_error, report_reason = "IdentityMismatch", "worker_not_bound" + else: + try: + report = _report_envelope(report_bytes) + except ProofError as error: + report_error, report_reason = error.code, "invalid_worker_report" + + for key, field, validate in (("grader", "grader", _grader), + ("samples", "samples", _samples)): + if report is None: + emit(key, "worker_report", "not_performed", reasons=[report_reason], + error=report_error) + continue + family = report[field] + if family is None: + emit(key, "worker_report", "not_performed", reasons=["report_family_incomplete"]) + continue + try: + validate(family) + except ProofError as error: + emit(key, "worker_report", "not_performed", reasons=["invalid_report_family"], + error=error.code) + continue + status, reasons = (family["status"], family["reasons"]) if key == "grader" else ( + "passed", [], + ) + payload = ({"claim_id": "C", "checker_revision": "1", **family} + if key == "grader" else {"samples": family}) + emit(key, "worker_report", status, {**candidate, **payload}, reasons) + + if facts["cleanup"] is None or facts["resource_leases"] is None: + emit("cleanup", "cleanup_capture", "not_performed", reasons=["cleanup_unobserved"]) + else: + confirmed = assessment._cleanup_confirmed(facts) + emit("cleanup", "cleanup_capture", "passed" if confirmed else "failed", + {**common, "confirmed": confirmed}, [] if confirmed else ["cleanup_unconfirmed"]) + return _snapshot(draft), dict(blobs) + + +def project(prepared, protocol_bytes, phase, facts, *, candidate_bytes, report_bytes): + """One-shot pure projection; hosts may cache ProofProjector for frozen config.""" + return ProofProjector(prepared, protocol_bytes).project( + phase, facts, candidate_bytes=candidate_bytes, report_bytes=report_bytes, + ) diff --git a/tests/check_installed.py b/tests/check_installed.py index 5b02b0d9..2a0c302d 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -791,6 +791,27 @@ def _inventory(root, prefix): [sys.executable, "-I", "-c", facade_probe], capture_output=True, text=True, check=False, ) assert facade.returncode == 0, facade.stderr +# The offline Benchmark port must ship in the wheel and remain independently importable. +assessment_probe = ( + "import sys\n" + "from inferrsi.benchmark.assessment import (\n" + " AssessmentService, assess, decode, encode, validate_assessment,\n" + ")\n" + "from inferrsi.benchmark.proofs import (\n" + " ProofProjector, decode_report, encode_report, project, validate_report,\n" + ")\n" + "assert all(callable(port) for port in\n" + " (AssessmentService, assess, decode, encode, validate_assessment,\n" + " ProofProjector, decode_report, encode_report, project, validate_report))\n" + "for prefix in ('inferrsi.apps', 'inferrsi.harness', 'inferrsi.knowledge'):\n" + " assert not [name for name in sys.modules if name == prefix\n" + " or name.startswith(prefix + '.')], prefix\n" +) +assessment_port = subprocess.run( + [sys.executable, "-I", "-B", "-c", assessment_probe], + capture_output=True, text=True, check=False, +) +assert assessment_port.returncode == 0, assessment_port.stderr # #452 M1: the MindIE adapter ships in the wheel. A fresh isolated interpreter imports the # package initializer with no core loaded and without a re-exported version, then imports # the stdio MCP module through its absolute package path. diff --git a/tests/test_benchmark_assessment.py b/tests/test_benchmark_assessment.py new file mode 100644 index 00000000..2a1e8638 --- /dev/null +++ b/tests/test_benchmark_assessment.py @@ -0,0 +1,632 @@ +"""The production offline port over retained P1 fixtures; no evaluation worker is run.""" + +import json +import subprocess +import sys +from copy import deepcopy +from hashlib import sha256 +from io import BytesIO +from itertools import count +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from inferrsi.apps import api as apps_api +from inferrsi.apps.handoff.bridge import check_assessment_identity, to_evidence +from inferrsi.apps.storage.archive import read_prepared +from inferrsi.apps.storage.artifact_reader import ArtifactReader +from inferrsi.benchmark import assessment +from inferrsi.benchmark import preparation as preparation_port +from inferrsi.contracts.json_v1 import CanonicalError, decode, encode +from inferrsi.harness.errors import PublicError +from tests.test_benchmark_preparation import preparation as preparation + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = ROOT / "docs/contracts/fixtures/t108-v4" +NOW = "2099-01-01T00:00:00Z" + + +def test_pure_codec_preserves_the_frozen_documentary_result(monkeypatch): + data = (GOLDEN / "assessment.json").read_bytes() + result = decode(data) + monkeypatch.setattr(Path, "open", lambda *a, **k: pytest.fail("pure codec opened a file")) + assert assessment.encode(result) == data + assert assessment.decode(data) == result + assessment.validate_assessment(result) + check_assessment_identity(result) + + +@pytest.fixture +def retained(preparation, tmp_path): + service, request, _, _, _, config, directory = preparation + output = apps_api.prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=preparation_port.encode, + validate_prepared=preparation_port.validate_prepared, + ) + assert output.result["exit_code"] == 0, output.result + prepared_ref = output.result["prepared_ref"] + prepared = read_prepared(config, prepared_ref) + record = decode((GOLDEN / "record-payload.json").read_bytes()) + record.update( + prepared_ref=prepared_ref, protocol_digest=prepared["protocol_digest"], + started=True, phase="finished", process_outcome="exited", exit_code=0, + elapsed_seconds="1", checks=[], artifacts=[], errors=[], + cleanup={"status": "confirmed", "process_exit_confirmed": True, + "termination_requested": False, "reasons": []}, + ) + record["steps"][0].update(status="exited", started_at=NOW, ended_at=NOW, + elapsed_seconds="1", exit_code=0, reasons=[]) + record["target_binding"] = { + "target_id": record["target_request"]["target_id"], "config_revision": "1", + "executor_version": "1", "control_domain": "fixture-domain", "capability_checks": [], + "runtime_bindings_by_step": [{ + "step_id": "main", "runtime_role": "local-python", "executable": "python", + "entrypoint": prepared["recipe"]["entrypoint"], "entrypoint_revision": "1", + "code_sha256": prepared["recipe"]["code_sha256"], "argv": [], + "environment_identity": None, + }], + } + serial = count() + + def put(label, data): + name = f"retained/{next(serial)}-{label}" + path = directory / name + path.parent.mkdir(exist_ok=True) + path.write_bytes(data) + return {"location": "archives/local/" + name, "sha256": sha256(data).hexdigest(), + "size_bytes": len(data), "media_type": "application/json"} + + candidate = put("candidate.txt", b"retained software-test candidate\n") + record["bound_inputs"] = [{ + "role": "candidate", "retained_ref": candidate, "actual_sha256": candidate["sha256"], + "snapshot_format": "file-v1", "captured_at": NOW, "attempt_id": record["attempt_id"], + "step_ids": ["main"], "materialized_path": "inputs/candidate.txt", + "expected_sha256": candidate["sha256"], "identity_status": "matched", "reasons": [], + }] + + def add(key, status="passed", **changes): + requirement = next(x for x in prepared["required_evidence"] if x["check_id"] == key) + proof = {"schema_version": 1, "attempt_id": record["attempt_id"], + "operation": "evaluate", "validation_scope": "p1"} + if key != "cleanup": + proof.update(candidate_id=record["candidate"]["candidate_id"], + actual_sha256=candidate["sha256"]) + if key in {"format", "grader"}: + proof.update(claim_id=requirement["for_claims"][0], checker_revision="1", + status=status, reasons=[] if status == "passed" else ["fixture_rejection"]) + elif key == "samples": + proof["samples"] = [ + {"sample_id": name, "role": role, "round": round_, "value": "1", "unit": "ms"} + for name, role, round_ in [("A1", "baseline", 1), ("B1", "candidate", 1), + ("B2", "candidate", 2), ("A2", "baseline", 2)] + ] + else: + proof["confirmed"] = True + proof.update(changes) + ref = put(key + ".json", encode(proof)) + kind = requirement["allowed_producer_kinds"][0] + check = { + **{k: requirement[k] for k in ("check_id", "checker_revision", "subject", "phase", + "artifact_schema")}, + "attempt_id": record["attempt_id"], "operation": "evaluate", "validation_scope": "p1", + "producer": {"kind": kind, "identity": "fixture_worker" if kind == "task_worker" + else "fixture-harness", "revision": "1", + "control_domain": "fixture-domain", + "capture_route": "retained-software-fixture"}, + "status": status, "evidence_refs": [ref], "reasons": [], + } + record["checks"].append(check) + return check, proof + + for key in ("format", "grader", "samples", "cleanup"): + add(key) + + def seal(): + ref = {"attempt_id": record["attempt_id"], "record_version": record["record_version"], + "artifact": put("record.json", encode(record))} + return ref, to_evidence(prepared, prepared_ref, ref, record) + + return {"prepared": prepared, "record": record, "config": config, "directory": directory, + "put": put, "add": add, "seal": seal, "workspace": tmp_path} + + +def run(case, *, reader=None, clock=lambda: NOW): + _, evidence = case["seal"]() + result = assessment.AssessmentService(clock=clock).assess( + case["prepared"], evidence, artifacts=reader or ArtifactReader(case["config"]), + ) + return result + + +def remove(case, *keys): + case["record"]["checks"] = [c for c in case["record"]["checks"] if c["check_id"] not in keys] + + +def test_all_positive_proofs_are_validated_against_captured_bytes(retained): + result = run(retained) + assert "code" not in result, result + assert [c["verdict"] for c in result["claims"]] == ["PASS"] * 3 + assert result["eligibility"][0]["status"] == "eligible" + assessment.validate_assessment(result) + check_assessment_identity(result) + assert assessment.decode(assessment.encode(result)) == result + schema = json.loads((ROOT / "docs/contracts/schemas/contract-v1.schema.json").read_text( + encoding="utf-8", + )) + Draft202012Validator({**schema, "$ref": "#/$defs/EvaluationResult"}).validate(result) + _, evidence = retained["seal"]() + Draft202012Validator({**schema, "$ref": "#/$defs/EvidenceInput"}).validate(evidence) + + +def test_t35_three_independent_claims_and_apps_exit_codes(retained): + remove(retained, "grader", "samples", "cleanup") + retained["add"]("grader", "failed") + ref, _ = retained["seal"]() + service = assessment.AssessmentService(clock=lambda: NOW) + for claim, exit_code in (("F", 0), ("C", 8), ("M", 9)): + workspace = retained["workspace"] / claim + workspace.mkdir() + output = apps_api.assess( + {"schema_version": 1, "record_ref": ref, + "primary_scope": "p1", "primary_claim_id": claim}, + workspace=workspace, storage_config=retained["config"], + assess_benchmark=service.assess, validate_assessment=assessment.validate_assessment, + encode_assessment=assessment.encode, + ) + assert output.result["exit_code"] == exit_code, output.result + assert output.result["assessment_ref"] is not None + assert [c["verdict"] for c in output.result["assessment"]["claims"]] == [ + "PASS", "FAIL", "INCONCLUSIVE", + ] + + +@pytest.mark.parametrize("kind", ["UnsafeArtifactPath", "ArtifactUnavailable", + "ArtifactIntegrityError", "bad_bytes"]) +def test_each_read_failure_is_only_a_claim_gap(retained, kind): + target = retained["record"]["checks"][0]["evidence_refs"][0] + reader = ArtifactReader(retained["config"]) + + class Fault: + def open(self, ref): + if ref == target: + if kind == "bad_bytes": + return BytesIO(b"not the retained bytes") + raise PublicError(kind, stage="open", message="fixture rejection") + return reader.open(ref) + + result = run(retained, reader=Fault()) + assert [c["verdict"] for c in result["claims"]] == ["INCONCLUSIVE", "PASS", "PASS"] + assert result["claims"][0]["missing_evidence"] == ["format"] + + +@pytest.mark.parametrize("reverse", [False, True], ids=["valid-first", "gap-first"]) +@pytest.mark.parametrize("kind,separate_check", [ + (kind, separate) for kind in ("UnsafeArtifactPath", "ArtifactUnavailable", + "ArtifactIntegrityError", "bad_bytes", "invalid_json") + for separate in (False, True) +] + [("empty", True)]) +def test_good_proof_cannot_hide_another_declared_evidence_gap( + retained, separate_check, reverse, kind, +): + good = retained["record"]["checks"][0] + good_refs = deepcopy(good["evidence_refs"]) + other, _ = retained["add"]("format", "failed") + target = other["evidence_refs"][0] + if kind == "invalid_json": + target = retained["put"]("invalid-proof.json", b"{}") + other["evidence_refs"] = [target] + if kind == "empty": + other["evidence_refs"] = [] + if not separate_check: + good["evidence_refs"].extend(other["evidence_refs"]) + retained["record"]["checks"].remove(other) + if reverse: + good["evidence_refs"].reverse() + elif reverse: + retained["record"]["checks"].reverse() + reader = ArtifactReader(retained["config"]) + + class Fault: + def open(self, ref): + if ref == target: + if kind in {"UnsafeArtifactPath", "ArtifactUnavailable", "ArtifactIntegrityError"}: + raise PublicError(kind, stage="open", message="fixture rejection") + if kind == "bad_bytes": + return BytesIO(b"not the retained failed proof") + return reader.open(ref) + + result = run(retained, reader=Fault()) + assert [c["verdict"] for c in result["claims"]] == ["INCONCLUSIVE", "PASS", "PASS"] + assert result["claims"][0]["missing_evidence"] == ["format"] + assert result["claims"][0]["evidence"] == good_refs + assert result["eligibility"][0]["status"] == "unknown" + + +@pytest.mark.parametrize("field,value", [ + ("record_ref", {}), ("claims", [{}]), ("comparison", [None]), + ("candidate", {"context_refs": None}), ("metrics", None), +]) +def test_public_identity_reports_malformed_nested_projection_as_declared_error( + retained, field, value, +): + result = run(retained) + result[field] = value + with pytest.raises(assessment.AssessmentError) as error: + assessment.assessment_id(result) + assert error.value.code == "InvalidRequest" + + +@pytest.mark.parametrize("error", [TypeError("broken host adapter"), + CanonicalError("host_adapter", "broken host adapter")], + ids=["type", "canonical"]) +def test_host_programming_errors_are_not_disguised_as_missing_evidence(retained, error): + class Broken: + def open(self, ref): + raise error + + with pytest.raises(type(error), match="broken host"): + run(retained, reader=Broken()) + + +@pytest.mark.parametrize("field,value", [("kind", "caller_assertion"), + ("control_domain", "untrusted-domain")]) +def test_untrusted_self_assertions_cannot_pass(retained, field, value): + retained["record"]["checks"][0]["producer"][field] = value + assert run(retained)["claims"][0]["verdict"] == "INCONCLUSIVE" + + +def test_proof_candidate_hash_is_not_its_own_identity_authority(retained): + remove(retained, "format") + retained["add"]("format", actual_sha256="0" * 64) + assert run(retained)["claims"][0]["verdict"] == "INCONCLUSIVE" + + +def test_v21_conflicting_proofs_are_aggregated_once(retained): + retained["add"]("format", "failed") + result = run(retained) + assert len(result["claims"]) == 3 + assert result["claims"][0]["verdict"] == "INCONCLUSIVE" + assert "conflicting_evidence" in result["claims"][0]["reasons"] + assert result["eligibility"][0]["status"] == "ineligible" + + +def test_running_snapshot_keeps_missing_terminal_evidence_inconclusive(retained): + remove(retained, "grader", "samples", "cleanup") + retained["record"].update(phase="running", process_outcome="unknown", exit_code=None) + retained["record"]["cleanup"].update(status="unknown", process_exit_confirmed=None) + before = encode(retained["record"]) + result = run(retained) + assert [c["verdict"] for c in result["claims"]] == ["PASS", "INCONCLUSIVE", "INCONCLUSIVE"] + assert result["claims"][1]["missing_evidence"] == ["grader"] + assert "completed_measurement" in result["claims"][2]["missing_evidence"] + assert encode(retained["record"]) == before + + +@pytest.mark.parametrize("mutation", ["order", "zero", "unit", "missing", "cleanup", "lease"]) +def test_measurement_requires_complete_sampling_cleanup_and_release(retained, mutation): + if mutation in {"order", "zero", "unit", "missing"}: + old = next(c for c in retained["record"]["checks"] if c["check_id"] == "samples") + with ArtifactReader(retained["config"]).open(old["evidence_refs"][0]) as stream: + samples = decode(stream.read())["samples"] + remove(retained, "samples") + if mutation == "order": + samples.reverse() + elif mutation == "zero": + samples[0]["value"] = "0" + elif mutation == "unit": + samples[0]["unit"] = "second" + else: + samples.pop() + retained["add"]("samples", samples=samples) + elif mutation == "cleanup": + retained["record"]["cleanup"]["process_exit_confirmed"] = None + else: + retained["record"]["resource_leases"] = { + "applicability": "applicable", "reasons": [], "leases": [{ + "lease_id": "lease-1", "resource_id": "cpu", "control_domain": "fixture-domain", + "owner": "fixture", "acquired_at": NOW, "release_requested_at": NOW, + "release_confirmed": False, "release_confirmed_at": None, "reasons": [], + }], + } + assert run(retained)["claims"][2]["verdict"] == "INCONCLUSIVE" + + +def test_binding_mismatch_is_refused_before_any_artifact_open(retained): + _, evidence = retained["seal"]() + evidence["record_digest"] = "0" * 64 + + class Never: + def open(self, ref): + pytest.fail("identity mismatch reached artifact IO") + + result = assessment.assess(retained["prepared"], evidence, artifacts=Never()) + assert result["code"] == "IdentityMismatch" + assert result["stage"] == "assessment_binding" + assert "assessment_id" not in result + + +@pytest.mark.parametrize("revision", ["grader", "claim-rule"]) +def test_v22_unsupported_same_family_rule_revision_is_refused(retained, revision): + prepared = retained["prepared"] + reader = ArtifactReader(retained["config"]) + with reader.open(prepared["protocol"]) as stream: + protocol = decode(stream.read()) + if revision == "grader": + protocol["grader"]["revision"] = "2" + prepared["grader"]["revision"] = "2" + else: + protocol["claim_definitions"][0]["decision_rule"]["revision"] = "2" + prepared["protocol"] = retained["put"]("unsupported-protocol.json", encode(protocol)) + prepared["protocol_digest"] = preparation_port.protocol_digest(protocol) + prepared["prepared_id"] = preparation_port.prepared_id(prepared) + record = retained["record"] + record["protocol_digest"] = prepared["protocol_digest"] + record["prepared_ref"]["prepared_id"] = prepared["prepared_id"] + record["prepared_ref"]["artifact"] = retained["put"]( + "unsupported-prepared.json", encode(prepared), + ) + result = run(retained) + assert result["code"] == "UnsupportedRuleVersion" + + +def test_outer_prepared_relocation_keeps_semantic_binding(retained): + before = retained["prepared"]["prepared_id"] + reader = ArtifactReader(retained["config"]) + ref = retained["prepared"]["protocol"] + with reader.open(ref) as stream: + ref["location"] = retained["put"]("relocated-protocol.json", stream.read())["location"] + assert preparation_port.prepared_id(retained["prepared"]) == before + retained["record"]["prepared_ref"]["artifact"] = retained["put"]( + "relocated-prepared.json", encode(retained["prepared"]), + ) + assert all(c["verdict"] == "PASS" for c in run(retained)["claims"]) + + +def test_v23_clock_and_typed_location_projection(retained): + result = run(retained) + later = run(retained, clock=lambda: "2099-01-01T00:00:01Z") + # Each seal creates a new locator but has identical fixed payload bytes. + assert result["assessment_id"] == later["assessment_id"] + assert assessment.encode(result) != assessment.encode(later) + changed = deepcopy(result) + changed["record_ref"]["artifact"]["location"] = "elsewhere/record.json" + check_assessment_identity(changed) + assessment.validate_assessment(changed) + changed["claims"][0]["verdict"] = "FAIL" + with pytest.raises(ValueError): + assessment.validate_assessment(changed) + + +def test_v23_fixed_clock_is_byte_identical_in_an_isolated_process(retained, tmp_path): + _, evidence = retained["seal"]() + reader = ArtifactReader(retained["config"]) + expected = assessment.AssessmentService(clock=lambda: NOW).assess( + retained["prepared"], evidence, artifacts=reader, + ) + blobs = {} + for ref in expected["evidence_artifacts"]: + with reader.open(ref) as stream: + blobs[ref["location"]] = stream.read().hex() + case = tmp_path / "case.json" + case.write_bytes(encode({ + "prepared": retained["prepared"], "evidence": evidence, "blobs": blobs, + })) + child = """ +import sys +from io import BytesIO +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from inferrsi.benchmark.assessment import AssessmentService, encode +from inferrsi.contracts.json_v1 import decode +value = decode(Path(sys.argv[2]).read_bytes()) +class Reader: + def open(self, ref): + return BytesIO(bytes.fromhex(value['blobs'][ref['location']])) +result = AssessmentService(clock=lambda: '2099-01-01T00:00:00Z').assess( + value['prepared'], value['evidence'], artifacts=Reader()) +sys.stdout.buffer.write(encode(result)) +""" + done = subprocess.run([sys.executable, "-I", "-c", child, str(ROOT / "src"), str(case)], + cwd=tmp_path, capture_output=True, check=True, timeout=30) + assert done.stdout == assessment.encode(expected) + + +@pytest.mark.parametrize("primary", [None, "unknown", "missing"]) +def test_t39_bad_primary_is_rejected_before_the_assessor(retained, primary): + ref, _ = retained["seal"]() + + def never(*args, **kwargs): + pytest.fail("bad primary reached Benchmark") + + request = {"schema_version": 1, "record_ref": ref, + "primary_scope": "p1", "primary_claim_id": primary} + if primary == "missing": + del request["primary_claim_id"] + result = apps_api.assess( + request, + workspace=retained["workspace"], storage_config=retained["config"], + assess_benchmark=never, validate_assessment=assessment.validate_assessment, + encode_assessment=assessment.encode, + ).result + assert result["exit_code"] == 2 and result["reason_code"] == "InvalidRequest" + + +def test_t39_missing_known_claim_and_failed_archive(retained, monkeypatch): + ref, _ = retained["seal"]() + request = {"schema_version": 1, "record_ref": ref, + "primary_scope": "p1", "primary_claim_id": "F"} + service = assessment.AssessmentService(clock=lambda: NOW) + + def omitted(*args, **kwargs): + result = service.assess(*args, **kwargs) + result["claims"].pop(0) + return result + + options = {"workspace": retained["workspace"], "storage_config": retained["config"], + "validate_assessment": assessment.validate_assessment, + "encode_assessment": assessment.encode} + result = apps_api.assess(request, assess_benchmark=omitted, **options).result + assert result["exit_code"] == 2 and result["reason_code"] == "AssessorError" + assert result["assessment_ref"] is None + + def failed(*args, **kwargs): + raise PublicError( + "ArchivePersistenceError", stage="assessment_archive", message="fixture IO", + ) + + monkeypatch.setattr(apps_api, "archive_assessment", failed) + result = apps_api.assess(request, assess_benchmark=service.assess, **options).result + assert result["exit_code"] == 6 and result["assessment_ref"] is None + assert result["assessment"] is not None + + +def test_assessment_does_not_launch_workers_or_access_the_network(retained, monkeypatch): + import socket + + def forbidden(*args, **kwargs): + pytest.fail("offline assessor attempted execution or network access") + + monkeypatch.setattr(subprocess, "Popen", forbidden) + monkeypatch.setattr(socket, "socket", forbidden) + assert all(c["verdict"] == "PASS" for c in run(retained)["claims"]) + + +@pytest.mark.parametrize("source", ["wrong-code", "foreign-check", "uncaptured", "not-started"]) +def test_evidence_cannot_override_actual_runtime_and_input_capture(retained, source): + record = retained["record"] + if source == "wrong-code": + record["target_binding"]["runtime_bindings_by_step"][0]["code_sha256"] = "0" * 64 + expected = ["PASS", "INCONCLUSIVE", "INCONCLUSIVE"] + elif source == "foreign-check": + record["checks"][0]["attempt_id"] = "foreign-attempt" + expected = ["INCONCLUSIVE", "PASS", "PASS"] + elif source == "uncaptured": + record["bound_inputs"][0]["actual_sha256"] = "0" * 64 + expected = ["INCONCLUSIVE"] * 3 + else: + record.update(started=False, process_outcome="not_started", exit_code=None) + record["steps"][0].update(status="not_started", started_at=None, ended_at=None, + exit_code=None, elapsed_seconds=None) + expected = ["PASS", "INCONCLUSIVE", "INCONCLUSIVE"] + assert [c["verdict"] for c in run(retained)["claims"]] == expected + + +def test_conflicting_measurements_are_not_first_or_last_wins(retained): + first = next(c for c in retained["record"]["checks"] if c["check_id"] == "samples") + with ArtifactReader(retained["config"]).open(first["evidence_refs"][0]) as stream: + samples = decode(stream.read())["samples"] + samples[0]["value"] = "2" + retained["add"]("samples", samples=samples) + result = run(retained) + assert result["claims"][2]["verdict"] == "INCONCLUSIVE" + assert "conflicting_evidence" in result["claims"][2]["reasons"] + assert result["eligibility"][0]["status"] == "ineligible" + + +def test_typed_projection_is_independent_of_python_object_aliasing(retained): + result = run(retained) + shared = result["claims"][0]["evidence"][0] + result["evidence_artifacts"].append(shared) + result["candidate"]["context_refs"] = [shared] + result["assessment_id"] = assessment.assessment_id(result) + assessment.validate_assessment(result) + check_assessment_identity(result) + + +@pytest.mark.parametrize("mutation", ["duplicate", "unknown-claim", "extra-field", "metric"]) +def test_pure_validator_rejects_invalid_results_even_with_recomputed_identity(retained, mutation): + result = run(retained) + if mutation == "duplicate": + result["claims"][1] = deepcopy(result["claims"][0]) + elif mutation == "unknown-claim": + result["claims"][0]["claim_id"] = "unknown" + elif mutation == "extra-field": + result["claims"][0]["overall"] = "PASS" + else: + result["metrics"] = [{"raw_samples": []}] + result["assessment_id"] = assessment.assessment_id(result) + with pytest.raises(ValueError): + assessment.validate_assessment(result) + + +def test_pure_timestamp_validation_checks_calendar_and_preserves_fractional_precision(retained): + result = run(retained) + result["assessed_at"] = "2099-02-29T00:00:00Z" + with pytest.raises(ValueError): + assessment.validate_assessment(result) + result["assessed_at"] = "2096-02-29T00:00:00.123456789Z" + assert assessment.decode(assessment.encode(result)) == result + + +@pytest.mark.parametrize("duplicate", [False, True]) +def test_raw_evidence_keeps_original_bytes_and_rejects_duplicate_json_keys(retained, duplicate): + check = retained["record"]["checks"][0] + with ArtifactReader(retained["config"]).open(check["evidence_refs"][0]) as stream: + proof = decode(stream.read()) + text = json.dumps(proof, indent=2) + if duplicate: + text = text[:-1] + ', "status": "passed"}' + raw = text.encode("utf-8") + check["evidence_refs"] = [retained["put"]("noncanonical-proof.json", raw)] + result = run(retained) + assert result["claims"][0]["verdict"] == ("INCONCLUSIVE" if duplicate else "PASS") + + +@pytest.mark.parametrize("character", ["\x00", "\x1f", "\x7f"]) +def test_pure_codec_rejects_control_characters_in_typed_paths(retained, character): + result = run(retained) + result["record_ref"]["artifact"]["location"] = "invalid/" + character + "/record.json" + with pytest.raises(ValueError): + assessment.encode(result) + + +def test_host_reader_cannot_change_the_snapshotted_input_during_assessment(retained): + _, evidence = retained["seal"]() + reader = ArtifactReader(retained["config"]) + digest = retained["prepared"]["protocol_digest"] + + class Mutating: + def open(self, ref): + retained["prepared"]["protocol_digest"] = "sha256:" + "0" * 64 + evidence["record"]["phase"] = "running" + return reader.open(ref) + + result = assessment.AssessmentService(clock=lambda: NOW).assess( + retained["prepared"], evidence, artifacts=Mutating(), + ) + assert result["protocol_digest"] == digest + assert all(c["verdict"] == "PASS" for c in result["claims"]) + + +def test_confirmed_execution_does_not_require_fabricating_unknown_clock_facts(retained): + retained["record"]["steps"][0].update(started_at=None, ended_at=None) + assert all(c["verdict"] == "PASS" for c in run(retained)["claims"]) + assert retained["record"]["steps"][0]["started_at"] is None + + +def test_invalid_prepared_resource_path_is_rejected_without_reading(retained): + _, evidence = retained["seal"]() + retained["prepared"]["resources"][0]["artifact"]["location"] = "invalid/\x00/resource" + + class Never: + def open(self, ref): + pytest.fail("invalid input reached artifact IO") + + result = assessment.assess(retained["prepared"], evidence, artifacts=Never()) + assert result["code"] == "InvalidRequest" + + +def test_conflicting_declared_candidate_expectation_is_a_binding_error(retained): + prepared, record = retained["prepared"], retained["record"] + prepared["expected_candidate"] = { + "candidate_id": record["candidate"]["candidate_id"], "sha256": "0" * 64, + "format_schema": deepcopy(prepared["candidate_contract"]["format_schema"]), + } + prepared["prepared_id"] = preparation_port.prepared_id(prepared) + record["prepared_ref"].update(prepared_id=prepared["prepared_id"], artifact=retained["put"]( + "different-expectation.json", preparation_port.encode(prepared), + )) + assert run(retained)["code"] == "IdentityMismatch" diff --git a/tests/test_benchmark_proofs.py b/tests/test_benchmark_proofs.py new file mode 100644 index 00000000..2673d033 --- /dev/null +++ b/tests/test_benchmark_proofs.py @@ -0,0 +1,386 @@ +"""Pure capture-draft tests; no worker, process observation or acceptance claim.""" + +import json +import socket +import subprocess +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path + +import pytest + +from inferrsi.benchmark import assessment, preparation, proofs +from inferrsi.contracts.json_v1 import decode, encode + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = ROOT / "docs/contracts/fixtures/t108-v4" + + +def report(): + return { + "schema_version": 1, "grader": {"status": "passed", "reasons": []}, + "samples": [{"sample_id": name, "role": role, "round": round_, "value": "1.25", + "unit": "ms"} for name, role, round_ in + (("A1", "baseline", 1), ("B1", "candidate", 1), + ("B2", "candidate", 2), ("A2", "baseline", 2))], + } + + +@pytest.fixture +def case(): + prepared_bytes = (GOLDEN / "prepared.json").read_bytes() + prepared = decode(prepared_bytes) + # Derive a new in-memory preparation; historical vector bytes are untouched. + prepared["expected_candidate"] = None + prepared["prepared_id"] = preparation.prepared_id(prepared) + prepared_bytes = preparation.encode(prepared) + protocol = (GOLDEN / "protocol.json").read_bytes() + data = b"captured candidate\n" + ref = {"location": "retained/candidate", "sha256": sha256(data).hexdigest(), + "size_bytes": len(data), "media_type": "text/plain"} + source = decode((GOLDEN / "record-payload.json").read_bytes()) + facts = {key: deepcopy(source[key]) for key in proofs.CONTEXT_FIELDS} + facts.update( + attempt_id="runtime-attempt-37", request_id="runtime-attempt-37", run_id="runtime-run-37", + request_digest="sha256:" + "a" * 64, + prepared_ref={"schema_version": 1, "prepared_id": prepared["prepared_id"], + "artifact": {"location": "retained/prepared", + "media_type": "application/json", + "sha256": sha256(prepared_bytes).hexdigest(), + "size_bytes": len(prepared_bytes)}}, + protocol_digest=prepared["protocol_digest"], candidate={ + "candidate_id": "runtime-candidate-37", "kind": "text", "input_role": "candidate", + "context_refs": [], "parent_id": None, "parent_digest": None, + }, + bound_inputs=[{"role": "candidate", "attempt_id": "runtime-attempt-37", + "retained_ref": ref, "actual_sha256": ref["sha256"], + "captured_at": "2099-01-01T00:00:00Z", "snapshot_format": "file-v1", + "step_ids": ["main"]}], + target_binding={"control_domain": "fixture-domain", "runtime_bindings_by_step": [{ + "step_id": "main", **{key: prepared["recipe"][key] for key in + ("runtime_role", "entrypoint", "entrypoint_revision", + "code_sha256")}, + }]}, + steps=[{"step_id": "main", "status": "exited", "started_at": "2099-01-01T00:00:00Z"}], + started=True, process_outcome="exited", exit_code=0, + cleanup={"status": "confirmed", "process_exit_confirmed": True}, + resource_leases={"applicability": "applicable", "leases": [{ + "release_confirmed": True, "release_confirmed_at": "2099-01-01T00:00:00Z", + }]}, + ) + return prepared, protocol, facts, data + + +def project(case, phase="finalizing", value=None, **kwargs): + prepared, protocol, facts, data = case + if phase == "preparing": + facts = deepcopy(facts) + facts.update(started=False, steps=[], process_outcome=None, exit_code=None, + cleanup=None, resource_leases=None) + kwargs.setdefault("candidate_bytes", data) + kwargs.setdefault("report_bytes", proofs.encode_report(report() if value is None else value)) + return proofs.project(prepared, protocol, phase, facts, **kwargs) + + +def by_id(draft): + return {check["check_id"]: check for check in draft["checks"]} + + +def test_report_codec_is_strict_but_preserves_input_whitespace_semantics(): + value = report() + assert proofs.decode_report(proofs.encode_report(value)) == value + assert proofs.decode_report(json.dumps(value, indent=2).encode()) == value + assert proofs.encode_report(value) == encode(value) + proofs.validate_report(value) + + +@pytest.mark.parametrize("phase", ["preparing", "finalizing"]) +def test_drafts_use_runtime_identity_and_existing_proof_schemas(case, phase): + prepared, _, facts, data = case + draft, blobs = project(case, phase) + assert draft["schema_version"] == 1 and draft["errors"] == [] + assert set(blobs) == ({"format"} if phase == "preparing" else {"grader", "samples", "cleanup"}) + required = {x["check_id"]: x for x in prepared["required_evidence"]} + for check in draft["checks"]: + assert "producer" not in check and "attempt_id" not in check + assert "evidence_refs" not in check and "artifact" not in check + raw = blobs[check["proof_key"]] + proof = assessment._proof(raw, required[check["check_id"]], facts, sha256(data).hexdigest()) + assert proof["attempt_id"] == "runtime-attempt-37" + if check["check_id"] != "cleanup": + assert proof["candidate_id"] == "runtime-candidate-37" + assert check["status"] == "passed" + + +@pytest.mark.parametrize("raw,code", [ + (b"{}", "InvalidRequest"), + (b'{"schema_version":1,"grader":null,"samples":null,"grader":null}', "InvalidRequest"), + (b'{"schema_version":2,"grader":null,"samples":null}', "UnsupportedSchema"), + (b'{"schema_version":true,"grader":null,"samples":null}', "UnsupportedSchema"), + (b'{"schema_version":1.0,"grader":null,"samples":null}', "InvalidRequest"), + (b'{"schema_version":1,"grader":NaN,"samples":null}', "InvalidRequest"), + (b'{"schema_version":1,"grader":"\\ud800","samples":null}', "InvalidRequest"), + (b"\xff", "InvalidRequest"), +]) +def test_bad_report_envelopes_refuse_without_fabricating_proofs(case, raw, code): + with pytest.raises(proofs.ProofError) as caught: + proofs.decode_report(raw) + assert caught.value.code == code + draft, blobs = project(case, report_bytes=raw) + assert by_id(draft)["grader"]["status"] == by_id(draft)["samples"]["status"] == "not_performed" + assert set(blobs) == {"cleanup"} + assert {x["code"] for x in draft["errors"]} == {code} + + +@pytest.mark.parametrize("extra", ["producer", "control_domain", "cleanup", "attempt_id", + "candidate_id", "actual_sha256", "record_ref"]) +def test_worker_cannot_supply_producer_identity_cleanup_or_references(case, extra): + value = report() + value[extra] = "forged" + draft, blobs = project(case, report_bytes=encode(value)) + assert set(blobs) == {"cleanup"} + assert all(by_id(draft)[key]["status"] == "not_performed" for key in ("grader", "samples")) + assert decode(blobs["cleanup"])["confirmed"] is True + + +@pytest.mark.parametrize("change", ["reversed", "duplicate", "float", "zero", "negative", + "noncanonical", "unit", "bool-round", "wrong-role", + "unknown-field", "missing", "null-item"]) +def test_invalid_samples_do_not_hide_an_independent_grader_result(case, change): + value = report() + sample = value["samples"][0] + if change == "reversed": + value["samples"].reverse() + elif change == "duplicate": + value["samples"][1] = deepcopy(sample) + elif change == "float": + sample["value"] = 1.25 + elif change == "zero": + sample["value"] = "0" + elif change == "negative": + sample["value"] = "-1" + elif change == "noncanonical": + sample["value"] = "1.0" + elif change == "unit": + sample["unit"] = "seconds" + elif change == "bool-round": + sample["round"] = True + elif change == "wrong-role": + sample["role"] = "candidate" + elif change == "unknown-field": + sample["producer"] = "forged" + elif change == "missing": + value["samples"].pop() + else: + value["samples"][0] = None + raw = json.dumps(value).encode() + draft, blobs = project(case, report_bytes=raw) + if change == "float": + # A forbidden JSON numeric token invalidates the raw envelope itself. + assert set(blobs) == {"cleanup"} + else: + assert set(blobs) == {"grader", "cleanup"} + assert by_id(draft)["grader"]["status"] == "passed" + assert by_id(draft)["samples"]["status"] == "not_performed" + with pytest.raises(proofs.ProofError): + proofs.decode_report(raw) + + +@pytest.mark.parametrize("grader", [{"status": "unknown", "reasons": []}, + {"status": "passed", "reasons": [None]}, None]) +def test_bad_or_incomplete_grader_keeps_valid_samples_and_cleanup(case, grader): + value = report() + value["grader"] = grader + draft, blobs = project(case, report_bytes=encode(value)) + assert set(blobs) == {"samples", "cleanup"} + assert by_id(draft)["grader"]["status"] == "not_performed" + assert by_id(draft)["samples"]["status"] == "passed" + + +def test_independent_failed_grade_is_not_rewritten_or_overlaid(case): + value = report() + value.update(grader={"status": "failed", "reasons": ["actual_grader_rejection"]}, samples=None) + draft, blobs = project(case, value=value) + assert by_id(draft)["grader"]["status"] == "failed" + assert decode(blobs["grader"])["reasons"] == ["actual_grader_rejection"] + assert by_id(draft)["samples"]["status"] == "not_performed" + + +@pytest.mark.parametrize("change", ["not-started", "entry-digest", "entry-name", "input-step", + "no-start-time", "bad-runtime-step"]) +def test_a_report_without_an_actual_bound_worker_is_not_trusted(case, change): + facts = case[2] + runtime = facts["target_binding"]["runtime_bindings_by_step"][0] + if change == "not-started": + facts["started"] = False + elif change == "entry-digest": + runtime["code_sha256"] = "b" * 64 + elif change == "entry-name": + runtime["entrypoint"] = "other_worker" + elif change == "input-step": + facts["bound_inputs"][0]["step_ids"] = ["other"] + elif change == "no-start-time": + facts["steps"][0]["started_at"] = None + else: + runtime["step_id"] = {} + draft, blobs = project(case) + assert set(blobs) == {"cleanup"} + assert {x["code"] for x in draft["errors"]} == {"IdentityMismatch"} + + +@pytest.mark.parametrize("change", ["attempt", "prepared", "protocol", "scope", "actual-sha", + "ref-sha", "ref-size", "capture-time", "duplicate-input"]) +def test_foreign_or_corrupt_capture_context_refuses(case, change): + facts = case[2] + bound = facts["bound_inputs"][0] + if change == "attempt": + bound["attempt_id"] = "foreign-attempt" + elif change == "prepared": + facts["prepared_ref"]["prepared_id"] = "sha256:" + "b" * 64 + elif change == "protocol": + facts["protocol_digest"] = "sha256:" + "b" * 64 + elif change == "scope": + facts["validation_scope"] = "foreign" + elif change == "actual-sha": + bound["actual_sha256"] = "b" * 64 + elif change == "ref-sha": + bound["retained_ref"]["sha256"] = "b" * 64 + elif change == "ref-size": + bound["retained_ref"]["size_bytes"] += 1 + elif change == "capture-time": + bound["captured_at"] = "not-a-time" + else: + facts["bound_inputs"].append(deepcopy(bound)) + with pytest.raises(proofs.ProofError) as error: + project(case) + assert error.value.code == "IdentityMismatch" + + +@pytest.mark.parametrize("phase", ["preparing", "finalizing"]) +def test_unsupported_actual_control_domain_refuses_each_affected_check(case, phase): + case[2]["target_binding"]["control_domain"] = "foreign-domain" + draft, blobs = project(case, phase) + assert blobs == {} and all(x["status"] == "not_performed" for x in draft["checks"]) + assert {x["code"] for x in draft["errors"]} == {"IdentityMismatch"} + + +@pytest.mark.parametrize("status,confirmed", [("confirmed", True), ("failed", False), + ("unknown", False)]) +def test_cleanup_proof_comes_from_controller_facts(case, status, confirmed): + case[2]["cleanup"]["status"] = status + draft, blobs = project(case) + assert decode(blobs["cleanup"])["confirmed"] is confirmed + assert by_id(draft)["cleanup"]["status"] == ("passed" if confirmed else "failed") + + +@pytest.mark.parametrize("change", ["release", "release-time", "process-exit", "unobserved"]) +def test_unknown_or_incomplete_cleanup_is_never_confirmed(case, change): + facts = case[2] + if change == "release": + facts["resource_leases"]["leases"][0]["release_confirmed"] = False + elif change == "release-time": + facts["resource_leases"]["leases"][0]["release_confirmed_at"] = None + elif change == "process-exit": + facts["cleanup"]["process_exit_confirmed"] = None + else: + facts["cleanup"] = None + draft, blobs = project(case) + if change == "unobserved": + assert by_id(draft)["cleanup"]["status"] == "not_performed" and "cleanup" not in blobs + else: + assert decode(blobs["cleanup"])["confirmed"] is False + + +def test_format_validation_reads_actual_bytes_and_can_fail_without_a_worker(case): + bad = b"\xff" + case[2]["bound_inputs"][0].update(actual_sha256=sha256(bad).hexdigest()) + case[2]["bound_inputs"][0]["retained_ref"].update(sha256=sha256(bad).hexdigest(), size_bytes=1) + case[2].update(started=False, steps=[], process_outcome=None, exit_code=None, + cleanup=None, resource_leases=None) + draft, blobs = project(case, "preparing", candidate_bytes=bad, report_bytes=None) + assert by_id(draft)["format"]["status"] == "failed" + assert decode(blobs["format"])["reasons"] == ["invalid_utf8"] + assert draft["errors"] == [] + + +@pytest.mark.parametrize("phase", ["preparing", "finalizing"]) +def test_missing_candidate_bytes_does_not_invent_a_positive_proof(case, phase): + draft, blobs = project(case, phase, candidate_bytes=None) + assert set(blobs) == (set() if phase == "preparing" else {"cleanup"}) + assert all(x["status"] == "not_performed" for x in draft["checks"] + if x["check_id"] != "cleanup") + + +def test_configuration_and_drafts_are_snapshotted_and_projection_is_pure(case, monkeypatch): + prepared, protocol, facts, data = case + frozen = deepcopy((prepared, facts)) + projector = proofs.ProofProjector(prepared, protocol) + prepared["required_evidence"].clear() + + def forbidden(*args, **kwargs): + pytest.fail("pure proof projection performed host I/O or execution") + + monkeypatch.setattr(Path, "open", forbidden) + monkeypatch.setattr(subprocess, "run", forbidden) + monkeypatch.setattr(socket, "create_connection", forbidden) + kwargs = {"candidate_bytes": data, "report_bytes": proofs.encode_report(report())} + first = projector.project("finalizing", facts, **kwargs) + clean = deepcopy(first) + first[0]["checks"][0]["artifact_schema"]["revision"] = "99" + first[1]["grader"] = b"mutated" + assert projector.project("finalizing", facts, **kwargs) == clean + assert facts == frozen[1] + + +@pytest.mark.parametrize("field", ["schema_version", "operation", "validation_scope", "grader"]) +def test_unsupported_frozen_configuration_refuses_before_projection(case, field): + prepared, protocol, _, _ = case + if field == "schema_version": + prepared[field] = 2 + elif field == "grader": + prepared[field]["revision"] = "2" + else: + prepared[field] = "other" + with pytest.raises(proofs.ProofError): + proofs.ProofProjector(prepared, protocol) + + +@pytest.mark.parametrize("argument", ["candidate_bytes", "report_bytes"]) +def test_unknown_host_content_types_propagate_as_programming_errors(case, argument): + with pytest.raises(TypeError): + project(case, **{argument: object()}) + + +@pytest.mark.parametrize("key", ["started", "process_outcome", "exit_code", "cleanup", + "resource_leases"]) +def test_a_late_context_cannot_be_relabelled_as_preparing(case, key): + prepared, protocol, facts, data = case + value = deepcopy(facts[key]) + facts.update(started=False, process_outcome=None, exit_code=None, cleanup=None, + resource_leases=None) + facts[key] = value + with pytest.raises(proofs.ProofError) as caught: + proofs.project(prepared, protocol, "preparing", facts, + candidate_bytes=data, report_bytes=None) + assert caught.value.code == "IdentityMismatch" + + +def test_a_frozen_expected_candidate_is_enforced(case): + prepared, protocol, facts, data = case + prepared["expected_candidate"] = { + "candidate_id": facts["candidate"]["candidate_id"], + "format_schema": {"name": "utf8-text", "revision": "1"}, + "sha256": sha256(data).hexdigest(), + } + prepared["prepared_id"] = preparation.prepared_id(prepared) + facts["prepared_ref"]["prepared_id"] = prepared["prepared_id"] + frozen_bytes = preparation.encode(prepared) + facts["prepared_ref"]["artifact"].update( + sha256=sha256(frozen_bytes).hexdigest(), size_bytes=len(frozen_bytes), + ) + projector = proofs.ProofProjector(prepared, protocol) + projector.project("finalizing", facts, candidate_bytes=data, report_bytes=encode(report())) + facts["candidate"]["candidate_id"] = "foreign-candidate" + with pytest.raises(proofs.ProofError) as caught: + projector.project("finalizing", facts, candidate_bytes=data, report_bytes=encode(report())) + assert caught.value.code == "IdentityMismatch"