From 79800f8b1ee5cd32413ef8592c738e3e54256731 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sat, 21 Nov 2026 23:45:26 +0800 Subject: [PATCH 1/4] feat(benchmark): define the pinned CPU evidence smoke task Keep the documentary fixture unchanged. Grade a captured answer against a fixed public byte-sum reference, and measure actual AB/BA operations independently of correctness, without performance thresholds. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/p1-cpu-smoke.md | 58 +++++++++++ src/inferrsi/benchmark/cpu_smoke.py | 97 +++++++++++++++++++ src/inferrsi/benchmark/preparation.py | 12 ++- src/inferrsi/benchmark/workers/__init__.py | 1 + .../benchmark/workers/fixture_worker.py | 66 +++++++++++++ 5 files changed, 230 insertions(+), 4 deletions(-) create mode 100644 docs/contracts/p1-cpu-smoke.md create mode 100644 src/inferrsi/benchmark/cpu_smoke.py create mode 100644 src/inferrsi/benchmark/workers/__init__.py create mode 100644 src/inferrsi/benchmark/workers/fixture_worker.py diff --git a/docs/contracts/p1-cpu-smoke.md b/docs/contracts/p1-cpu-smoke.md new file mode 100644 index 00000000..49ca255e --- /dev/null +++ b/docs/contracts/p1-cpu-smoke.md @@ -0,0 +1,58 @@ +# P1 CPU handoff/evidence smoke task + +The accepted [task specification](https://github.com/InferMatrixLab/InferRSI/issues/565#issuecomment-7064895332) +defines `p1-cpu-smoke@1` with `p1-cpu-smoke-rules@1`. This is a software +handoff/evidence task, not a model-quality, speedup or service benchmark. +The historical `p1-fixture@1 / p1-rules@1` behavior and vectors are unchanged. + +## Frozen operations and evidence + +The standalone `benchmark/workers/fixture_worker.py` contains the fixed public +`INPUT_BYTES`. Reference A sums those bytes and returns canonical ASCII decimal +plus one LF. Candidate B reads the actual candidate file's answer bytes. The +worker compares them exactly: equality gives C passed, otherwise C failed with +`response_mismatch`. A missing candidate read gives no C proof rather than a +mathematical failure. Candidate text cannot supply grader status or provenance. + +M measures four real operations A1/B1/B2/A2 (AB, BA), using monotonic nanoseconds +converted by integer arithmetic to canonical Decimal milliseconds. Sampling runs +regardless of C's result. There is no fill, clamp, minimum, warmup, retry or +re-measurement. A failed/non-positive/missing sample makes that family incomplete. +The initial C comparison is outside the four timed operations. Values are +evidence, with no numeric performance threshold or comparison/gain claim. + +F remains the separate H-owned capture of actual UTF-8 candidate bytes. Cleanup +and acquired-lease release remain H facts. The existing P1 F/C/M decision rules, +closed worker-report format, evidence matching, conflict handling and empty +metrics/comparison arrays are reused. A failed C does not overwrite M or skip its +sampling; an M PASS is measurement validity only. + +## Explicit configuration and resource closure + +`benchmark.cpu_smoke.worker_source()` explicitly reads the packaged owned worker. +`protocol(worker_bytes, *, control_domain)` is pure and requires the host's +declared actual domain. `prepare_service(control_domain=...)` snapshots that +resource and configures the existing production PreparationService. Its resource +capability serves immutable bytes for worker-code and baseline-code. The latter +is Protocol.baseline and identifies the same full source, including A and the +embedded input. No network, external input, device, model or undeclared script is +part of the task. The label/decimal parameters retain their encoding-only role. + +`default_request()` provides the supported task request. The host archives its +preparation, configures the B ProofProjector and Apps EvidenceCapture, and supplies +H with a trusted module_root containing those exact `fixture_worker.py` bytes. +The frozen argv remains candidate path and report path. H imports no B module. + +## Acceptance sequence and limits + +Record exact worker/source/input pins in the PR before running acceptance cases, +as required by the owner. Never retune them after results. Test the real correct +answer, valid UTF-8 wrong answer and invalid UTF-8 independently; verify actual +timing validity/order without value thresholds. Retain missing, rejected, +conflicting and failed cases and original-attempt replay outcomes. + +PR readiness is separate from shared production acceptance. The stacked codec, +capture, adapter and launcher prerequisites must reach main, branches must sync, +and final-head checks/two independent reviews must pass before integration. The +shared D023/t108-v4 matrix then runs on merged source. Tests on a development +stack do not establish that final acceptance or any service-performance claim. diff --git a/src/inferrsi/benchmark/cpu_smoke.py b/src/inferrsi/benchmark/cpu_smoke.py new file mode 100644 index 00000000..4fecb992 --- /dev/null +++ b/src/inferrsi/benchmark/cpu_smoke.py @@ -0,0 +1,97 @@ +"""The accepted CPU evidence-smoke profile; configuration reads only owned worker bytes.""" + +from copy import deepcopy +from hashlib import sha256 +from importlib.resources import files +from io import BytesIO + +from . import preparation + + +def worker_source() -> bytes: + """Explicit read-only resource port; no worker import or execution.""" + return files("inferrsi.benchmark.workers").joinpath("fixture_worker.py").read_bytes() + + +def protocol(worker_bytes: bytes, *, control_domain: str) -> dict: + """Build a protocol from supplied bytes, with no host I/O or guessed domain.""" + if not isinstance(worker_bytes, bytes): + raise TypeError("Expected worker source bytes.") + preparation._require(preparation._id(control_domain), "Invalid actual control domain.") + digest = sha256(worker_bytes).hexdigest() + pin = { + "artifact": {"location": "tasks/p1-cpu-smoke/fixture_worker.py", "sha256": digest, + "size_bytes": len(worker_bytes), "media_type": "text/x-python"}, + "format_schema": {"name": "utf8-text", "revision": "1"}, + "required_for": ["prepare", "execute"], + } + requirements = [] + for key, claim, phase, kinds in ( + ("format", "F", "preparing", ["harness_capture"]), + ("grader", "C", "finalizing", ["task_worker"]), + ("samples", "M", "running", ["harness_capture", "task_worker"]), + ("cleanup", "M", "finalizing", ["harness_capture"]), + ): + requirements.append({ + "check_id": key, "checker_revision": "1", "allowed_producer_kinds": kinds, + "subject": "candidate", "phase": phase, + "artifact_schema": {"name": key + "-proof", "revision": "1"}, + "matching_rule_revision": "1", "allowed_control_domains": [control_domain], + "for_claims": [claim], + }) + return { + "schema_version": 1, "task": {"name": "p1-cpu-smoke", "revision": "1"}, + "grader": {"name": "p1-cpu-smoke-rules", "revision": "1"}, + "operation": "evaluate", "validation_scope": "p1", + "candidate_contract": deepcopy(preparation._CANDIDATE), + "worker_recipe": { + "recipe_revision": "1", "runtime_role": "local-python", + "entrypoint_kind": "python_module", "entrypoint": "fixture_worker", + "entrypoint_revision": "1", "code_sha256": digest, + "args": ["{input:candidate}", "{output:grader}"], "input_roles": ["candidate"], + "outputs": [{"role": "grader", "path": "outputs/grader.json", "required": True, + "media_type": "application/json", + "format_schema": {"name": "fixture-grader", "revision": "1"}}], + "required_capabilities": ["local-process-tree"], + }, + "input_requirements": {"candidate_role": "candidate"}, + "claim_definitions": [{ + "claim_id": key, "validation_scope": "p1", + "decision_rule": {"name": "p1-" + key, "revision": "1"}, + "required_checks": checks, "requires_completed_measurement": key == "M", + "requires_cleanup_confirmed": key == "M", + } for key, checks in (("F", ["format"]), ("C", ["grader"]), + ("M", ["samples", "cleanup"]))], + "metric_definitions": [], "required_evidence": requirements, + "task_parameters": {"decimal_fixture": "1.25", "label": "CPU evidence smoke"}, + "baseline": {"role": "baseline-code", **deepcopy(pin)}, + "reference": None, "quality_reference": None, + "sampling": {"order": ["AB", "BA"], "repetitions": 2}, + "feedback_visibility": ["format"], "budget_policy_refs": [], + "resources": [{"role": "worker-code", **deepcopy(pin)}], + "display_notes": "CPU handoff/evidence smoke; no performance-gain or model-quality claim.", + } + + +def prepare_service(*, control_domain: str): + """Snapshot the packaged worker; close resources through the existing prepare port.""" + source = worker_source() + + def resources(role): + if role not in ("worker-code", "baseline-code"): + raise FileNotFoundError("Unknown task resource role.") + return BytesIO(source) + + return preparation.PreparationService( + protocol(source, control_domain=control_domain), resources=resources, + worker_role="worker-code", + ) + + +def default_request(): + return { + "schema_version": 1, "task": {"name": "p1-cpu-smoke", "revision": "1"}, + "operation": "evaluate", "validation_scope": "p1", "input_pins": [], + "task_parameters": {"decimal_fixture": "1.25", "label": "CPU evidence smoke"}, + "comparison_conditions": {}, "expected_candidate": None, "budget_policy_refs": [], + } diff --git a/src/inferrsi/benchmark/preparation.py b/src/inferrsi/benchmark/preparation.py index ddcd5ec0..48d34d96 100644 --- a/src/inferrsi/benchmark/preparation.py +++ b/src/inferrsi/benchmark/preparation.py @@ -213,13 +213,18 @@ def _requirements(requirements) -> None: "Unsupported frozen P1 evidence rule.", "UnsupportedRuleVersion") +def _supported_task(task, grader): + return any(task == {"name": name, "revision": "1"} + and grader == {"name": rules, "revision": "1"} for name, rules in + (("p1-fixture", "p1-rules"), ("p1-cpu-smoke", "p1-cpu-smoke-rules"))) + + def _protocol(protocol: dict) -> None: canonical_encode(protocol) _object(protocol, _PROTOCOL) _require(type(protocol["schema_version"]) is int and protocol["schema_version"] == 1, "Unsupported protocol version.", "UnsupportedSchema") - _require(protocol["task"] == {"name": "p1-fixture", "revision": "1"} - and protocol["grader"] == {"name": "p1-rules", "revision": "1"}, + _require(_supported_task(protocol["task"], protocol["grader"]), "Only the accepted P1 task and grader revision are supported.", "UnsupportedRuleVersion") _require(protocol["operation"] == "evaluate" and protocol["validation_scope"] == "p1", @@ -278,8 +283,7 @@ def _prepared(prepared: dict) -> None: _candidate(prepared["candidate_contract"]) _recipe(prepared["recipe"]) _requirements(prepared["required_evidence"]) - _require(prepared["task"] == {"name": "p1-fixture", "revision": "1"} - and prepared["grader"] == {"name": "p1-rules", "revision": "1"} + _require(_supported_task(prepared["task"], prepared["grader"]) and prepared["operation"] == "evaluate" and prepared["validation_scope"] == "p1", "Unsupported prepared task revision or scope.", "UnsupportedRuleVersion") _require(isinstance(prepared["display_notes"], str) diff --git a/src/inferrsi/benchmark/workers/__init__.py b/src/inferrsi/benchmark/workers/__init__.py new file mode 100644 index 00000000..d923381a --- /dev/null +++ b/src/inferrsi/benchmark/workers/__init__.py @@ -0,0 +1 @@ +"""Standalone Benchmark worker resources; importing the package executes no task.""" diff --git a/src/inferrsi/benchmark/workers/fixture_worker.py b/src/inferrsi/benchmark/workers/fixture_worker.py new file mode 100644 index 00000000..d0d770f5 --- /dev/null +++ b/src/inferrsi/benchmark/workers/fixture_worker.py @@ -0,0 +1,66 @@ +"""Standalone p1-cpu-smoke@1 worker; no inference quality or performance-gain claim.""" + +import json +import sys +import time +from pathlib import Path + +INPUT_BYTES = b"InferRSI CPU evidence smoke task\n" + + +def baseline(): + return str(sum(INPUT_BYTES)).encode("ascii") + b"\n" + + +def _milliseconds(nanoseconds): + # Integer conversion preserves canonical decimals without rounding or clamping. + return (f"{nanoseconds // 1_000_000}.{nanoseconds % 1_000_000:06d}" + .rstrip("0").rstrip(".")) + + +def _samples(candidate_path, clock): + rows, complete = [], True + for sample_id, role, round_ in (("A1", "baseline", 1), ("B1", "candidate", 1), + ("B2", "candidate", 2), ("A2", "baseline", 2)): + try: + start = clock() + baseline() if role == "baseline" else candidate_path.read_bytes() + end = clock() + except (OSError, ArithmeticError, ValueError): + complete = False + continue + if type(start) is not int or type(end) is not int or end <= start: + complete = False + continue + rows.append({"sample_id": sample_id, "role": role, "round": round_, + "value": _milliseconds(end - start), "unit": "ms"}) + return rows if complete else None + + +def evaluate(candidate_path, *, clock=None): + """Grade exact bytes, then measure all four operations independently of C.""" + candidate_path = Path(candidate_path) + try: + candidate = candidate_path.read_bytes() + except OSError: + grader = None + else: + matched = candidate == baseline() + grader = {"status": "passed" if matched else "failed", + "reasons": [] if matched else ["response_mismatch"]} + samples = _samples(candidate_path, time.monotonic_ns if clock is None else clock) + return {"schema_version": 1, "grader": grader, "samples": samples} + + +def main(): + if len(sys.argv) != 3: + return 2 + report = evaluate(sys.argv[1]) + output = Path(sys.argv[2]) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, ensure_ascii=False, allow_nan=False), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From d654646013226d655a394b2a2512891b85ad0b26 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sun, 22 Nov 2026 00:09:11 +0800 Subject: [PATCH 2/4] test(benchmark): freeze CPU task acceptance coverage Cover actual worker sampling, independent claims, retained reference failures and original-attempt replay without changing the pinned worker, input or criteria. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/apps-module-migrations.json | 2 + docs/operations.md | 1 + tests/check_installed.py | 10 +- tests/test_benchmark_cpu_smoke.py | 245 ++++++++++++++++++++++++++++++ 4 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 tests/test_benchmark_cpu_smoke.py diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index b3140f3d..dd0138d5 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -38,6 +38,7 @@ "tests/test_apps_publication.py", "tests/test_apps_replay.py", "tests/test_benchmark_assessment.py", + "tests/test_benchmark_cpu_smoke.py", "tests/test_benchmark_preparation.py", "tests/test_harness_entry_verification.py", "tests/test_result_delivery.py" @@ -66,6 +67,7 @@ "tests/test_apps_generation_plan.py", "tests/test_apps_module_migrations.py", "tests/test_benchmark_assessment.py", + "tests/test_benchmark_cpu_smoke.py", "tests/test_harness_entry_verification.py" ] }, diff --git a/docs/operations.md b/docs/operations.md index d9d9a206..2b59ee78 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -60,6 +60,7 @@ Effect classes used below: | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Assess one fixed P1 record offline | Python `benchmark.assessment.assess`; [production assess](contracts/production-assess.md) | Read-only retained evidence / pure encoding | Verifies the frozen protocol, actual capture and proof bytes; returns independent claims without launching a worker or writing an archive; Apps selects the primary key and archives the result | | Project P1 worker reports into capture drafts | Python `benchmark.proofs` codecs / `ProofProjector.project`; [proof drafts](contracts/production-proofs.md) | Pure computation | Produces proof bytes from supplied capture facts and reports; H must authenticate, retain and seal them before commit; no worker or grader execution | +| Configure the CPU evidence-smoke task | Python `benchmark.cpu_smoke.prepare_service`; [task contract](contracts/p1-cpu-smoke.md) | Read-only owned resource configuration | Freezes the standalone worker and same-source baseline for the existing prepare port; actual execution/retention stays in H; no performance-gain claim | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | | Retain an executed retrieval | Python `apps.workflows.campaign.save_retrieval`; [capture contract](knowledge-retrieval-trace.md#retain-a-standalone-result) | Local evidence-writing | Saves an existing catalog/collect result under a fresh campaign capture ID; no new search or automatic publication | diff --git a/tests/check_installed.py b/tests/check_installed.py index 3bab5da7..802ffda4 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -806,7 +806,15 @@ def _inventory(root, prefix): assert facade.returncode == 0, facade.stderr # The offline Benchmark port must ship in the wheel and remain independently importable. assessment_probe = ( - "import sys\n" + "import hashlib, sys\n" + "from inferrsi.benchmark.cpu_smoke import default_request, prepare_service, worker_source\n" + "source = worker_source()\n" + "configured = prepare_service(control_domain='fixture-domain')\n" + "p, data, resources = configured.prepare(default_request())\n" + "assert p['task'] == {'name': 'p1-cpu-smoke', 'revision': '1'}\n" + "assert p['recipe']['code_sha256'] == hashlib.sha256(source).hexdigest()\n" + "with resources('baseline-code') as stream:\n" + " assert stream.read() == source\n" "from inferrsi.benchmark.assessment import (\n" " AssessmentService, assess, decode, encode, validate_assessment,\n" ")\n" diff --git a/tests/test_benchmark_cpu_smoke.py b/tests/test_benchmark_cpu_smoke.py new file mode 100644 index 00000000..f8393123 --- /dev/null +++ b/tests/test_benchmark_cpu_smoke.py @@ -0,0 +1,245 @@ +"""Pinned CPU software-task mechanics; not merged production or performance acceptance.""" + +import sqlite3 +import subprocess +import sys +from copy import deepcopy +from decimal import Decimal +from hashlib import sha256 +from itertools import count +from pathlib import Path + +import pytest + +from inferrsi.apps import api +from inferrsi.apps.storage import archive as archive_port +from inferrsi.apps.storage.artifact_reader import ArtifactReader +from inferrsi.benchmark import assessment, cpu_smoke, proofs +from inferrsi.benchmark import preparation as prepare_port +from inferrsi.benchmark.workers import fixture_worker as worker +from inferrsi.contracts.json_v1 import decode +from inferrsi.harness import execution_service, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from tests.test_benchmark_preparation import preparation as preparation + +ROOT = Path(__file__).resolve().parents[1] + + +def candidates(): + # Task-defined expected answer, independent of the worker's baseline() implementation. + return {"correct": str(sum(worker.INPUT_BYTES)).encode("ascii") + b"\n", + "wrong": b"wrong\n", "invalid-utf8": b"\xff\n"} + + +def check_samples(value): + assert value is not None + assert [(x["sample_id"], x["role"], x["round"]) for x in value] == [ + ("A1", "baseline", 1), ("B1", "candidate", 1), + ("B2", "candidate", 2), ("A2", "baseline", 2), + ] + assert all(x["unit"] == "ms" and Decimal(x["value"]) > 0 for x in value) + proofs.validate_report({"schema_version": 1, "grader": None, "samples": value}) + + +@pytest.mark.parametrize("kind", ["correct", "wrong", "invalid-utf8"]) +def test_packaged_worker_runs_without_package_imports_and_measures_real_operations(tmp_path, kind): + path, output = tmp_path / "candidate.txt", tmp_path / "report.json" + path.write_bytes(candidates()[kind]) + result = subprocess.run([sys.executable, "-I", "-B", worker.__file__, str(path), str(output)], + capture_output=True, timeout=10) + assert result.returncode == 0, result.stderr + value = proofs.decode_report(output.read_bytes()) + assert value["grader"]["status"] == ("passed" if kind == "correct" else "failed") + check_samples(value["samples"]) + + +def test_wrong_answer_still_executes_all_four_samples_in_order(tmp_path, monkeypatch): + path = tmp_path / "candidate.txt" + path.write_bytes(candidates()["wrong"]) + original_read, original_baseline = Path.read_bytes, worker.baseline + actions = [] + + def read(value): + actions.append("B") + return original_read(value) + + def reference(): + actions.append("A") + return original_baseline() + + monkeypatch.setattr(Path, "read_bytes", read) + monkeypatch.setattr(worker, "baseline", reference) + clock = count(step=1000) + result = worker.evaluate(path, clock=lambda: next(clock)) + assert result["grader"]["status"] == "failed" + assert actions == ["B", "A", "A", "B", "B", "A"] + check_samples(result["samples"]) + + +@pytest.mark.parametrize("values", [[0] * 8, list(range(8, 0, -1)), [True] * 8]) +def test_invalid_timing_keeps_grade_but_never_fills_or_retries_samples(tmp_path, values): + path = tmp_path / "candidate.txt" + path.write_bytes(candidates()["correct"]) + clock = iter(values) + reads = [] + + def next_time(): + reads.append(True) + return next(clock) + + result = worker.evaluate(path, clock=next_time) + assert result["grader"]["status"] == "passed" and result["samples"] is None + assert len(reads) == 8 + + +def test_missing_candidate_produces_incomplete_families(tmp_path): + result = worker.evaluate(tmp_path / "missing") + assert result == {"schema_version": 1, "grader": None, "samples": None} + + +@pytest.mark.parametrize("nanoseconds,text", [(1, "0.000001"), (1000, "0.001"), + (1_000_000, "1"), (10_000_000, "10")]) +def test_integer_unit_conversion_preserves_canonical_values(nanoseconds, text): + assert worker._milliseconds(nanoseconds) == text + + +def test_profile_freezes_complete_code_and_baseline_resource_closure(): + source = cpu_smoke.worker_source() + service = cpu_smoke.prepare_service(control_domain="fixture-domain") + prepared, protocol_bytes, reader = service.prepare(cpu_smoke.default_request()) + prepare_port.validate_prepared(prepared, protocol_bytes) + protocol = decode(protocol_bytes) + assert protocol["baseline"]["artifact"]["sha256"] == sha256(source).hexdigest() + assert prepared["recipe"]["code_sha256"] == sha256(source).hexdigest() + assert {item["role"] for item in prepared["resources"]} == {"worker-code", "baseline-code"} + for role in ("worker-code", "baseline-code"): + with reader(role) as stream: + assert stream.read() == source + + +@pytest.mark.parametrize("kind", ["task-version", "grader-version", "mixed-pair"]) +def test_unknown_or_mixed_task_versions_are_not_guessed(kind): + protocol = cpu_smoke.protocol(cpu_smoke.worker_source(), control_domain="fixture-domain") + if kind == "task-version": + protocol["task"]["revision"] = "2" + elif kind == "grader-version": + protocol["grader"]["revision"] = "2" + else: + protocol["grader"] = {"name": "p1-rules", "revision": "1"} + with pytest.raises(prepare_port.PreparationError) as rejected: + prepare_port.PreparationService(protocol, resources=lambda role: b"", + worker_role="worker-code") + assert rejected.value.code == "UnsupportedRuleVersion" + + +@pytest.mark.parametrize("kind,publish_failure,missing", [ + ("correct", False, False), ("wrong", False, False), ("invalid-utf8", False, False), + ("correct", True, False), ("correct", False, True), +]) +def test_real_cpu_task_has_independent_claims_fixed_refs_and_original_attempt_replay( + preparation, tmp_path, monkeypatch, kind, publish_failure, missing, +): + _, _, _, _, _, config, directory = preparation + service = cpu_smoke.prepare_service(control_domain=execution_service._CONTROL_DOMAIN) + prepared_output = api.prepare( + {"schema_version": 1, "request": cpu_smoke.default_request()}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=prepare_port.encode, + validate_prepared=prepare_port.validate_prepared, output_id="cpu-prepare", + ) + assert prepared_output.result["exit_code"] == 0, prepared_output.result + prepared_ref = prepared_output.result["prepared_ref"] + prepared = archive_port.read_prepared(config, prepared_ref) + candidate = candidates()[kind] + if not missing: + (directory / "candidate.txt").write_bytes(candidate) + ref = {"location": "archives/local/candidate.txt", "media_type": "text/plain", + "sha256": sha256(candidate).hexdigest(), "size_bytes": len(candidate)} + vector = decode((ROOT / "tests/fixtures/harness-prepared-binding/d023-null-timeout.json") + .read_bytes()) + vector.update(attempt_id="runtime-cpu-attempt", request_id="runtime-cpu-attempt", + run_id="runtime-cpu-run") + proposal = {key: deepcopy(vector[key]) + for key in ("candidate", "inputs", "generation_record_ref")} + proposal["candidate"]["candidate_id"] = "runtime-cpu-candidate" + proposal["inputs"][0].update(source=ref["location"], source_ref=ref, + expected_sha256=ref["sha256"]) + request = {"schema_version": 1, "prepared_ref": prepared_ref, + "attempt": {key: vector[key] for key in execution_service.ATTEMPT_FIELDS}, + "proposal": proposal, "output_root": vector["output_root"]} + reader = ArtifactReader(config) + with reader.open(prepared["protocol"]) as stream: + projector = proofs.ProofProjector(prepared, stream.read()) + projections = [] + + def project(*args, **kwargs): + projections.append(args[0]) + return projector.project(*args, **kwargs) + + capture = api.EvidenceCapture(project, candidate_role="candidate", report_role="grader") + store = run_store.RunStore(tmp_path / "store", "test") + target = LocalCpuTarget("fixture-cpu", "1", work_root=tmp_path / "work", + module_root=Path(worker.__file__).parent) + harness = Harness(store, target, artifact_reader=reader, evidence_capture=capture) + + def publication_error(*args, **kwargs): + raise OSError("synthetic output publication failure") + + with monkeypatch.context() as fault: + if publish_failure: + fault.setattr(archive_port, "_publish_directory", publication_error) + executed = api.execute(request, workspace=tmp_path, storage_config=config, + execute_harness=harness.execute, output_id="cpu-execute") + assert executed.result["exit_code"] == (6 if publish_failure else 3 if missing else 0) + record, record_ref = executed.result["execution"], executed.result["record_ref"] + assert record["started"] is (not missing) + if publish_failure: + assert executed.result["error"]["latest_record_ref"] == record_ref + if not missing: + assert projections == ["preparing", "finalizing"] + assert record["cleanup"]["status"] == "confirmed" and record["exit_code"] == 0 + raw = next(x for x in record["artifacts"] if x["role"] == "grader") + with reader.open(raw["artifact"]) as stream: + report = proofs.decode_report(stream.read()) + check_samples(report["samples"]) + assert {x["check_id"] for x in record["checks"]} >= { + "format", "grader", "samples", "cleanup", + } + else: + assert projections == [] + assessed = api.assess( + {"schema_version": 1, "record_ref": record_ref, "primary_scope": "p1", + "primary_claim_id": "C"}, workspace=tmp_path, storage_config=config, + assess_benchmark=assessment.assess, encode_assessment=assessment.encode, + validate_assessment=assessment.validate_assessment, output_id="cpu-assess", + ) + assert assessed.result["assessment"] is not None, assessed.result + result = assessed.result["assessment"] + expected = (["INCONCLUSIVE"] * 3 if missing else + ["FAIL" if kind == "invalid-utf8" else "PASS", + "PASS" if kind == "correct" else "FAIL", "PASS"]) + assert [x["verdict"] for x in result["claims"]] == expected + assert result["task"] == {"name": "p1-cpu-smoke", "revision": "1"} + assert result["metrics"] == result["comparison"] == [] + view = api.read_attempt_view( + record_ref, storage_config=config, assessment_ref=assessed.result["assessment_ref"], + include_trace=True, + ) + assert view["record_ref"] == result["record_ref"] == record_ref + assert archive_port.read_result(tmp_path, "cpu-assess").result == assessed.result + + def forbidden(*args, **kwargs): + pytest.fail("fixed CPU task replay implicitly launched or captured again") + + if missing: + (directory / "candidate.txt").write_bytes(candidate) + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + retired = Harness(store, LocalCpuTarget("retired", "2", work_root=tmp_path / "retired"), + artifact_reader=object(), evidence_capture=object()) + replay = api.execute(request, workspace=tmp_path, storage_config=config, + execute_harness=retired.execute, output_id="cpu-replay") + assert replay.result["record_ref"] == record_ref + assert len(projections) == (0 if missing else 2) + with sqlite3.connect(store.root / run_store.INDEX_NAME) as connection: + assert connection.execute("SELECT count(*) FROM attempts").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM records").fetchone()[0] == 1 From bebd7bf44899edf18e84ea70c3368245ac6a7dc2 Mon Sep 17 00:00:00 2001 From: zengchuang Date: Sun, 22 Nov 2026 00:52:08 +0800 Subject: [PATCH 3/4] fix(benchmark): use the monotonic performance counter for sampling Preserve the input, operations, order and validity rules. Keep the coarse-clock run inconclusive and freeze the new source before its fixed cases. Agent-assisted-by: Codex (author) Signed-off-by: zengchuang --- docs/contracts/p1-cpu-smoke.md | 4 +++- src/inferrsi/benchmark/workers/fixture_worker.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/contracts/p1-cpu-smoke.md b/docs/contracts/p1-cpu-smoke.md index 49ca255e..d8745c45 100644 --- a/docs/contracts/p1-cpu-smoke.md +++ b/docs/contracts/p1-cpu-smoke.md @@ -14,7 +14,9 @@ worker compares them exactly: equality gives C passed, otherwise C failed with `response_mismatch`. A missing candidate read gives no C proof rather than a mathematical failure. Candidate text cannot supply grader status or provenance. -M measures four real operations A1/B1/B2/A2 (AB, BA), using monotonic nanoseconds +M measures four real operations A1/B1/B2/A2 (AB, BA), using `perf_counter_ns` +monotonic nanoseconds under the accepted +[clock implementation revision](https://github.com/InferMatrixLab/InferRSI/pull/569#issuecomment-7065967644), converted by integer arithmetic to canonical Decimal milliseconds. Sampling runs regardless of C's result. There is no fill, clamp, minimum, warmup, retry or re-measurement. A failed/non-positive/missing sample makes that family incomplete. diff --git a/src/inferrsi/benchmark/workers/fixture_worker.py b/src/inferrsi/benchmark/workers/fixture_worker.py index d0d770f5..cd0f2c57 100644 --- a/src/inferrsi/benchmark/workers/fixture_worker.py +++ b/src/inferrsi/benchmark/workers/fixture_worker.py @@ -48,7 +48,7 @@ def evaluate(candidate_path, *, clock=None): matched = candidate == baseline() grader = {"status": "passed" if matched else "failed", "reasons": [] if matched else ["response_mismatch"]} - samples = _samples(candidate_path, time.monotonic_ns if clock is None else clock) + samples = _samples(candidate_path, time.perf_counter_ns if clock is None else clock) return {"schema_version": 1, "grader": grader, "samples": samples} From 2e203ae6f1810fe9e52f09d5c1b51f086465e93a Mon Sep 17 00:00:00 2001 From: zengchuang-hw Date: Sun, 22 Nov 2026 00:59:43 +0800 Subject: [PATCH 4/4] fix(benchmark): validate the accepted CPU task results Agent-assisted-by: Codex (author) Signed-off-by: zengchuang-hw --- src/inferrsi/benchmark/assessment.py | 3 +-- tests/test_benchmark_cpu_smoke.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/inferrsi/benchmark/assessment.py b/src/inferrsi/benchmark/assessment.py index c04e376b..d7b4ec85 100644 --- a/src/inferrsi/benchmark/assessment.py +++ b/src/inferrsi/benchmark/assessment.py @@ -118,8 +118,7 @@ def validate_assessment(result: dict) -> None: _object(result, _RESULT) _require(type(result["schema_version"]) is int and result["schema_version"] == 1, code="UnsupportedSchema") - _require(result["task"] == {"name": "p1-fixture", "revision": "1"} - and result["grader"] == {"name": "p1-rules", "revision": "1"} + _require(preparation._supported_task(result["task"], result["grader"]) and result["assessor"] == {"name": "fixture-assessor", "revision": "1"}, code="UnsupportedRuleVersion") _timestamp(result["assessed_at"]) diff --git a/tests/test_benchmark_cpu_smoke.py b/tests/test_benchmark_cpu_smoke.py index f8393123..53611887 100644 --- a/tests/test_benchmark_cpu_smoke.py +++ b/tests/test_benchmark_cpu_smoke.py @@ -132,6 +132,25 @@ def test_unknown_or_mixed_task_versions_are_not_guessed(kind): assert rejected.value.code == "UnsupportedRuleVersion" +@pytest.mark.parametrize("kind", ["task-version", "grader-version", "mixed-pair", "assessor"]) +def test_result_validator_rejects_unregistered_task_grader_or_assessor(kind): + result = decode((ROOT / "docs/contracts/fixtures/t108-v4/assessment.json").read_bytes()) + result.update(task={"name": "p1-cpu-smoke", "revision": "1"}, + grader={"name": "p1-cpu-smoke-rules", "revision": "1"}) + if kind == "task-version": + result["task"]["revision"] = "2" + elif kind == "grader-version": + result["grader"]["revision"] = "2" + elif kind == "mixed-pair": + result["grader"] = {"name": "p1-rules", "revision": "1"} + else: + result["assessor"]["revision"] = "2" + result["assessment_id"] = assessment.assessment_id(result) + with pytest.raises(assessment.AssessmentError) as rejected: + assessment.validate_assessment(result) + assert rejected.value.code == "UnsupportedRuleVersion" + + @pytest.mark.parametrize("kind,publish_failure,missing", [ ("correct", False, False), ("wrong", False, False), ("invalid-utf8", False, False), ("correct", True, False), ("correct", False, True),