diff --git a/docs/validation/restart-source-restore-20261120.md b/docs/validation/restart-source-restore-20261120.md new file mode 100644 index 0000000..d02d87b --- /dev/null +++ b/docs/validation/restart-source-restore-20261120.md @@ -0,0 +1,43 @@ +# Pinned source restoration preparation + +The recovered census refers to an original source snapshot whose historical +directory is absent locally. Its source members have been recovered privately. +The maintained census verifier requires the original binding; changing the frozen +manifest to point elsewhere would change the evidence being verified. + +The new [restoration helper](../../scripts/adhoc/restore_restart_source_20261120.py) +resolves EdgeMech through RepositorySettings and the non-Mech KG-Microbe source +owner through the maintained corpus resolver and exact-origin validator. Its +read-only plan independently passed all three source members' original hashes, +byte counts, row counts, header and selected-prefix checks. Their combined size +is 3,267,015,185 bytes. The [public digest mapping](../../scripts/adhoc/provenance/restart-source-20261120/plan-summary.json) +binds both retained plan versions without exposing private operational paths. + +Live restoration remains separate and unexecuted at this checkpoint. It requires +independent final review, complete validation, a reviewed PR and exact published +head. The apply path refuses any existing destination, checks source-owner state +again under a short lease, and preserves failed reservations. It copies the +verified members through guarded regular-file reads and checks persisted bytes. +The original manifest is written under a pending name, flushed and hash-checked; +directory flushes precede an atomic, non-replacing publication. No source host, +provider, native worker or prompt is invoked by this helper. + +Independent review identified two defects before live apply, recorded in +[#231](https://github.com/CultureBotAI/EdgeMech/issues/231): a direct manifest +write could expose a partial completion marker, and a replaced FIFO could block +a plain input read while the metadata lease was held. The repaired version uses +atomic publication and the maintained no-follow/nonblocking regular-file reader +for source and persisted inputs. The original executed read-only plan helper +and receipt remain privately unchanged under the provenance policy in +[#228](https://github.com/CultureBotAI/EdgeMech/issues/228). + +Sixteen offline tests cover partial marker writes, fsync and directory-flush +failure, concurrent markers, changed bytes, existing/symlink destinations, +nonregular source and readback, lock refusal, changed source-owner state, and +context release after copy failure. Fresh-process regression runs against the +retained pre-fix bytes are private evidence; they never invoke a live restoration. +Complete-suite and final-review results accompany the exact PR candidate. + +Restored transport bytes would still require full maintained census replay, +exact occurrence accounting and exhaustive prior-assessment reconciliation. +Scientific coverage remains zero and A/B counts remain unresolved. diff --git a/scripts/adhoc/provenance/restart-source-20261120/plan-summary.json b/scripts/adhoc/provenance/restart-source-20261120/plan-summary.json new file mode 100644 index 0000000..04b9851 --- /dev/null +++ b/scripts/adhoc/provenance/restart-source-20261120/plan-summary.json @@ -0,0 +1,23 @@ +{ + "date_utc": "2026-11-20", + "purpose": "Verified source-restoration plan; full census replay still pending", + "issues": ["CultureBotAI/EdgeMech#228", "CultureBotAI/EdgeMech#231"], + "original_plan_helper_sha256": "4d1217614f522da3ae8e6d254249c6e6e4aadaa6a71047be247068aa47bf9774", + "original_plan_receipt_sha256": "ade94ea9badd1d25855fc55ddcc1a74f87c55f45f3746764bb5935102bb66379", + "repaired_plan_helper_sha256": "a06782633c92ce8ce59957a0d65c371592e2bdafbd7ad6f52836f8bd4b9cdb37", + "repaired_plan_receipt_sha256": "3159950bf4d085614a47e913dfa831ba6b436eee3fe9d87d6f8aa689b9ce82b6", + "original_source_manifest_sha256": "f585e55a616d3771f382c91c78713cb8d74812c7ba177576090ccd5f9e6654b2", + "original_census_manifest_sha256": "a2b8353349816fd067ab3a7112fd1c2f5ee9c02971591ea52f31e9b981cbdd2e", + "verified_source_members": 3, + "verified_source_member_bytes": 3267015185, + "verified_edge_member_sha256": "e6a846112ff8c769a8daa8a3c509c3390bd7b5e341c7998ce9188ebece9d4c9a", + "verification": "Maintained source verifier checked all original member hashes, bytes, row counts, header and selected prefix at the recovered location; no manifest rewrite", + "focused_tests_passed": 16, + "focused_log_sha256": "417add8ce523b530ebf6659426bf3831a752641770b7d7f5ae8314db0d5d7e58", + "live_apply_executed": false, + "full_census_replay_completed": false, + "scientific_review_coverage": 0, + "queue_A_count": null, + "queue_B_count": null, + "private_preservation": "Executed plan helper bytes, receipts, failure tests and operational bindings remain private; original pins and source bytes unchanged" +} diff --git a/scripts/adhoc/restore_restart_source_20261120.py b/scripts/adhoc/restore_restart_source_20261120.py new file mode 100644 index 0000000..4073d82 --- /dev/null +++ b/scripts/adhoc/restore_restart_source_20261120.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Restore a recovered, pinned source snapshot without overwriting existing data. + +Purpose: enable the maintained full census verifier at its original bound path. +Inputs: private recovery receipt, census manifest, CLAW root and fresh result root. +Outputs: private plan/attempt receipts and, only with --apply, the absent original +source snapshot. No source-host access, credential transfer, Git commit or launch. +UTC date: 2026-11-20. Issues: CultureBotAI/EdgeMech#228 (helper provenance), +CultureBotAI/EdgeMech#231 (atomic publication and guarded-read review findings). +Provenance: consumes parent-owned recovery; original bootstrap and recovered bytes +remain unchanged. This is transport restoration, with scientific review coverage 0. +All operational paths/bindings are private inputs and receipts, never public logs. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +from edgemech.census import _fsync_directory, _metadata, _read_handle, _snapshot, _verify_source +from edgemech.storage.parquet import file_hash, safe_path + +CENSUS_SHA = "a2b8353349816fd067ab3a7112fd1c2f5ee9c02971591ea52f31e9b981cbdd2e" +SOURCE_SHA = "f585e55a616d3771f382c91c78713cb8d74812c7ba177576090ccd5f9e6654b2" + + +def write_json(path: Path, value: dict) -> None: + with path.open("x", encoding="utf-8") as stream: + json.dump(value, stream, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def no_symlinks(path: Path) -> Path: + path = path.absolute() + if any(p.is_symlink() for p in (path, *path.parents)): + raise ValueError("snapshot paths must not traverse symlinks") + return path + + +def publish_manifest(destination: Path, manifest_bytes: bytes, expected: str) -> None: + """Publish only flushed, hash-verified bytes; retain pending bytes on failure.""" + pending = destination / ".source-manifest.pending" + marker = destination / "manifest.json" + with pending.open("xb") as stream: + stream.write(manifest_bytes) + stream.flush() + os.fsync(stream.fileno()) + with _read_handle(pending) as stream: + readback = stream.read(len(manifest_bytes) + 1) + if readback != manifest_bytes or hashlib.sha256(readback).hexdigest() != expected: + raise ValueError("pending manifest readback differs") + _fsync_directory(destination) + _fsync_directory(destination.parent) + os.link(pending, marker, follow_symlinks=False) # Atomic, never replaces a marker. + _fsync_directory(destination) + with _read_handle(marker) as stream: + if stream.read(len(manifest_bytes) + 1) != manifest_bytes: + raise ValueError("published manifest readback differs") + pending.unlink() # Success only; failed reservations and pending bytes are retained. + _fsync_directory(destination) + + +def restore_absent(source: Path, destination: Path, manifest: dict, + manifest_sha: str, *, seconds: float = 120) -> dict: + """Called under the owner's short lease; leaves failed reservations intact.""" + no_symlinks(source) + no_symlinks(destination) + if destination.exists(): + raise FileExistsError("source destination exists; inspect without overwriting") + if not destination.parent.is_dir(): + raise ValueError("snapshot parent must already exist") + names = list(manifest["members"]) + if any(Path(name).name != name for name in names): + raise ValueError("this bounded restore supports direct regular snapshot members only") + if "manifest.json" in names: + raise ValueError("manifest cannot also be a data member") + expected = {name: value["sha256"] for name, value in manifest["members"].items()} + manifest_bytes, _ = _metadata(source / "manifest.json") + if hashlib.sha256(manifest_bytes).hexdigest() != manifest_sha: + raise ValueError("original source manifest bytes changed") + deadline = time.monotonic() + seconds + destination.mkdir(mode=0o700) # Exclusive reservation; never replace a failed attempt. + copied = [] + for name in names: + if time.monotonic() >= deadline: + raise TimeoutError("bounded source restoration deadline exceeded") + original = safe_path(source / name) + digest = hashlib.sha256() + count = 0 + with _read_handle(original) as incoming, (destination / name).open("xb") as outgoing: + while block := incoming.read(4 * 1024 * 1024): + if time.monotonic() >= deadline: + raise TimeoutError("bounded source restoration deadline exceeded") + outgoing.write(block) + digest.update(block) + count += len(block) + outgoing.flush() + os.fsync(outgoing.fileno()) + if digest.hexdigest() != expected[name]: + raise ValueError(f"copy input hash changed: {name}") + if count != manifest["members"][name]["bytes"]: + raise ValueError(f"copy input size changed: {name}") + readback = hashlib.sha256() + with _read_handle(destination / name) as stream: + while block := stream.read(4 * 1024 * 1024): + if time.monotonic() >= deadline: + raise TimeoutError("bounded source readback deadline exceeded") + readback.update(block) + if readback.hexdigest() != expected[name]: + raise ValueError(f"persisted copy failed hash readback: {name}") + copied.append({"member": name, "bytes": count, "sha256": expected[name]}) + if time.monotonic() >= deadline: + raise TimeoutError("bounded source restoration deadline exceeded") + publish_manifest(destination, manifest_bytes, manifest_sha) + copied.append({"member": "manifest.json", "bytes": len(manifest_bytes), + "sha256": manifest_sha}) + return {"copied": copied, "manifest_published_last": True} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--claw-root", type=Path, required=True) + parser.add_argument("--recovery-receipt", type=Path, required=True) + parser.add_argument("--census", type=Path, required=True) + parser.add_argument("--results", type=Path, required=True) + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + result_root = no_symlinks(args.results) + result_root.mkdir(mode=0o700) # Fresh attempt, including plans. + record = { + "started_at_utc": datetime.now(timezone.utc).isoformat(), + "mode": "apply" if args.apply else "plan", + "helper_sha256": file_hash(Path(__file__)), + "scientific_review_coverage": 0, "status": "preparing", + } + try: + claw = no_symlinks(args.claw_root) + sys.path[:0] = [str(claw), str(claw / "src")] + from kg_microbe_fleet.roots import resolve_kg_microbe_root + from plugins.lock_manager import LockManager + from plugins.repository_settings import ( + RepositorySettings, + RepositoryTarget, + merged_repository_environment, + ) + + env = merged_repository_environment(claw / ".env") + settings = RepositorySettings.from_environment(environ=env) + edge = settings.get_target("edgemech") + kg_root = resolve_kg_microbe_root(claw_root=claw, environ=env) + if kg_root is None: + raise ValueError("source-owner repository is not configured") + # KG-Microbe is the input corpus, not a Mech manifest row. Use its maintained + # resolver and the same RepositoryTarget exact-origin validator. + target = RepositoryTarget("kg-microbe", kg_root, "Knowledge-Graph-Hub/kg-microbe") + repo = target.open() + git_before = repo.git.status("--porcelain=v1", "--untracked-files=all") + receipt_bytes, recovery = _metadata(args.recovery_receipt) + census_bytes, census = _metadata(args.census) + if hashlib.sha256(census_bytes).hexdigest() != CENSUS_SHA: + raise ValueError("census differs from the accepted full-population reference") + if recovery.get("state") != "PRIVATE_STAGING_TRANSFER_AND_BYTE_READBACK_COMPLETE": + raise ValueError("parent recovery is not complete") + if recovery.get("census_manifest_sha256") != CENSUS_SHA: + raise ValueError("recovery receipt census binding differs") + archive = recovery["source_archive"] + recovered = no_symlinks(Path(archive["extracted_directory"])) + raw_manifest, manifest, _ = _snapshot(recovered, SOURCE_SHA) + if census["source"]["manifest_sha256"] != SOURCE_SHA: + raise ValueError("census source pin differs") + if archive["manifest_sha256"] != SOURCE_SHA: + raise ValueError("recovery source pin differs") + if archive["members"] != { + name: {"bytes": item["bytes"], "sha256": item["sha256"]} + for name, item in manifest["members"].items() + }: + raise ValueError("parent source member bindings differ") + destination = no_symlinks(Path(census["source"]["snapshot_path"])) + relative = destination.relative_to(kg_root) + if len(relative.parts) != 3 or relative.parts[:2] != ("data", "merged"): + raise ValueError("destination is not an original merged-snapshot directory") + if not re.fullmatch(r"[0-9]{8}", relative.name): + raise ValueError("snapshot directory must be the pinned YYYYMMDD") + if destination.exists(): + raise FileExistsError("original snapshot destination exists; no overwrite") + if repo.git.check_ignore(str(relative / "manifest.json")) == "": + raise ValueError("restored snapshot must remain ignored research data") + required_bytes = sum(item["bytes"] for item in manifest["members"].values()) + if shutil.disk_usage(destination.parent).free < required_bytes + 10 * 1024**3: + raise ValueError("insufficient space for source copy plus 10 GiB reserve") + # In-memory relocation for independent source validation only. Neither + # retained manifest is rewritten; full census replay will use original path. + selected_source_hash = _verify_source( + {**census["source"], "snapshot_path": str(recovered)}, manifest + ) + record.update({ + "status": "verified_plan", "canonical_edgemech": str(edge.path), + "source_owner": str(kg_root), "source_owner_head": repo.head.commit.hexsha, + "source": str(recovered), "destination": str(destination), + "recovery_receipt_sha256": hashlib.sha256(receipt_bytes).hexdigest(), + "census_sha256": CENSUS_SHA, "source_manifest_sha256": SOURCE_SHA, + "source_prefix_sha256": selected_source_hash, + "member_count": len(manifest["members"]), "member_bytes": required_bytes, + "original_manifest_bytes": len(raw_manifest), + }) + write_json(result_root / "plan.json", record) + if args.apply: + os.environ["OPENCLAW_ORCHESTRATION_ROOT"] = str(claw) + with LockManager().lock("kg-microbe", "restore-pinned-edge-census-source", timeout=900): + current = target.open() + if current.git.status("--porcelain=v1", "--untracked-files=all") != git_before: + raise ValueError("source owner worktree changed before restoration") + record["copy_readback"] = restore_absent(recovered, destination, manifest, SOURCE_SHA) + if target.open().git.status("--porcelain=v1", "--untracked-files=all") != git_before: + raise ValueError("source owner Git state changed during restoration") + record["status"] = "restored_source_bytes_verified_full_census_replay_pending" + record["finished_at_utc"] = datetime.now(timezone.utc).isoformat() + write_json(result_root / "receipt.json", record) + print(json.dumps({"status": record["status"], "scientific_review_coverage": 0})) + except BaseException as exc: + record.update(status="failed", error_type=type(exc).__name__, error=str(exc), + finished_at_utc=datetime.now(timezone.utc).isoformat()) + write_json(result_root / "failure.json", record) + raise + + +if __name__ == "__main__": + main() diff --git a/tests/test_restore_restart_source_20261120.py b/tests/test_restore_restart_source_20261120.py new file mode 100644 index 0000000..7bb05c4 --- /dev/null +++ b/tests/test_restore_restart_source_20261120.py @@ -0,0 +1,290 @@ +"""Failure preservation tests for source restoration, with tiny private fixtures.""" + +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = Path(os.environ.get( + "EDGE_RESTORE_HELPER_UNDER_TEST", ROOT / "scripts/adhoc/restore_restart_source_20261120.py" +)) +SPEC = importlib.util.spec_from_loader( + "restore_source", importlib.machinery.SourceFileLoader("restore_source", str(SOURCE)) +) +helper = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(helper) + + +@pytest.fixture +def snapshot(tmp_path): + source = tmp_path / "recovered" + source.mkdir() + data = b"subject\tpredicate\tobject\nA\tP\tB\n" + (source / "merged-kg_edges.tsv").write_bytes(data) + manifest = {"members": {"merged-kg_edges.tsv": { + "bytes": len(data), "rows": 1, "sha256": hashlib.sha256(data).hexdigest() + }}} + raw = json.dumps(manifest).encode() + (source / "manifest.json").write_bytes(raw) + return source, tmp_path / "original", manifest, hashlib.sha256(raw).hexdigest() + + +def test_exact_copy_preserves_manifest_bytes_and_publishes_it_last(snapshot, monkeypatch): + source, destination, manifest, digest = snapshot + original_open = Path.open + writes = [] + + def observe(path, mode="r", *args, **kwargs): + if path.parent == destination and mode == "xb": + writes.append(path.name) + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(Path, "open", observe) + result = helper.restore_absent(source, destination, manifest, digest) + assert writes == ["merged-kg_edges.tsv", ".source-manifest.pending"] + assert result["manifest_published_last"] + for name in ["merged-kg_edges.tsv", "manifest.json"]: + assert (destination / name).read_bytes() == (source / name).read_bytes() + + +def test_existing_destination_is_never_overwritten(snapshot): + source, destination, manifest, digest = snapshot + destination.mkdir() + (destination / "valuable").write_text("preserve") + with pytest.raises(FileExistsError): + helper.restore_absent(source, destination, manifest, digest) + assert sorted(p.name for p in destination.iterdir()) == ["valuable"] + assert (destination / "valuable").read_text() == "preserve" + + +def test_changed_member_retains_failed_attempt_without_manifest_and_refuses_retry(snapshot): + source, destination, manifest, digest = snapshot + (source / "merged-kg_edges.tsv").write_bytes(b"changed") + with pytest.raises(ValueError, match="input hash changed"): + helper.restore_absent(source, destination, manifest, digest) + assert (destination / "merged-kg_edges.tsv").read_bytes() == b"changed" + assert not (destination / "manifest.json").exists() + with pytest.raises(FileExistsError): + helper.restore_absent(source, destination, manifest, digest) + + +def test_changed_manifest_fails_before_destination_mutation(snapshot): + source, destination, manifest, digest = snapshot + (source / "manifest.json").write_bytes(b"{}") + with pytest.raises(ValueError, match="manifest bytes changed"): + helper.restore_absent(source, destination, manifest, digest) + assert not destination.exists() + + +def test_deadline_leaves_reserved_failed_attempt_without_completion_marker(snapshot): + source, destination, manifest, digest = snapshot + with pytest.raises(TimeoutError): + helper.restore_absent(source, destination, manifest, digest, seconds=0) + assert destination.is_dir() + assert not (destination / "manifest.json").exists() + + +def test_destination_symlink_is_rejected(snapshot): + source, destination, manifest, digest = snapshot + destination.symlink_to(source, target_is_directory=True) + with pytest.raises(ValueError, match="symlinks"): + helper.restore_absent(source, destination, manifest, digest) + assert (source / "manifest.json").is_file() + + +def test_partial_manifest_write_cannot_expose_completion_marker(snapshot, monkeypatch): + source, destination, manifest, digest = snapshot + original_open = Path.open + + @contextmanager + def partial_writer(path, mode, *args, **kwargs): + with original_open(path, mode, *args, **kwargs) as stream: + def write(data): + stream.write(data[:5]) + stream.flush() + raise OSError("injected partial manifest write") + yield SimpleNamespace(write=write) + + def inject(path, mode="r", *args, **kwargs): + if path.parent == destination and path.name in {"manifest.json", ".source-manifest.pending"}: + if mode == "xb": + return partial_writer(path, mode, *args, **kwargs) + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(Path, "open", inject) + with pytest.raises(OSError, match="partial manifest write"): + helper.restore_absent(source, destination, manifest, digest) + assert not (destination / "manifest.json").exists() + assert (destination / ".source-manifest.pending").read_bytes() + + +def test_manifest_fsync_failure_cannot_expose_completion_marker(snapshot, monkeypatch): + source, destination, manifest, digest = snapshot + real_fsync = os.fsync + + def inject(fd): + inode = os.fstat(fd).st_ino + for name in ("manifest.json", ".source-manifest.pending"): + path = destination / name + if path.exists() and path.stat().st_ino == inode: + raise OSError("injected manifest fsync failure") + return real_fsync(fd) + + monkeypatch.setattr(os, "fsync", inject) + with pytest.raises(OSError, match="manifest fsync failure"): + helper.restore_absent(source, destination, manifest, digest) + assert not (destination / "manifest.json").exists() + assert (destination / ".source-manifest.pending").is_file() + + +def test_directory_flush_failure_before_publish_has_no_marker(snapshot, monkeypatch): + source, destination, manifest, digest = snapshot + + def fail(path): + raise OSError("injected directory flush failure") + + monkeypatch.setattr(helper, "_fsync_directory", fail) + with pytest.raises(OSError, match="directory flush failure"): + helper.restore_absent(source, destination, manifest, digest) + assert not (destination / "manifest.json").exists() + assert (destination / ".source-manifest.pending").is_file() + + +def test_atomic_publication_never_replaces_concurrent_marker(snapshot, monkeypatch): + source, destination, manifest, digest = snapshot + real_link = os.link + + def race(src, dst, **kwargs): + Path(dst).write_bytes(b"concurrent marker") + return real_link(src, dst, **kwargs) + + monkeypatch.setattr(os, "link", race) + with pytest.raises(FileExistsError): + helper.restore_absent(source, destination, manifest, digest) + assert (destination / "manifest.json").read_bytes() == b"concurrent marker" + assert (destination / ".source-manifest.pending").is_file() + + +@pytest.mark.parametrize("name", ["merged-kg_edges.tsv", "manifest.json"]) +def test_nonregular_source_refuses_without_blocking(snapshot, name): + source, destination, manifest, digest = snapshot + path = source / name + path.unlink() + os.mkfifo(path) + code = """ +import importlib.machinery, importlib.util, json, sys +from pathlib import Path +spec = importlib.util.spec_from_loader('probe', importlib.machinery.SourceFileLoader('probe', sys.argv[1])) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +try: + module.restore_absent(Path(sys.argv[2]), Path(sys.argv[3]), json.loads(sys.argv[4]), sys.argv[5]) +except Exception as error: + assert 'regular file' in str(error), str(error) +else: + raise AssertionError('nonregular source accepted') +""" + subprocess.run([sys.executable, "-c", code, str(SOURCE), str(source), str(destination), + json.dumps(manifest), digest], check=True, timeout=10, capture_output=True) + assert not (destination / "manifest.json").exists() + + +def test_nonregular_persisted_readback_refuses(snapshot, monkeypatch): + source, destination, manifest, digest = snapshot + original_read = helper._read_handle + + def replace_before_read(path): + if Path(path) == destination / "merged-kg_edges.tsv": + Path(path).unlink() + os.mkfifo(path) + return original_read(path) + + monkeypatch.setattr(helper, "_read_handle", replace_before_read) + with pytest.raises(ValueError, match="regular file"): + helper.restore_absent(source, destination, manifest, digest) + assert not (destination / "manifest.json").exists() + + +@pytest.mark.parametrize("failure", ["lock_refusal", "owner_changed", "copy_failure"]) +def test_main_mutation_boundary_and_context_release(snapshot, monkeypatch, tmp_path, failure): + source, _, manifest, digest = snapshot + kg_root = tmp_path / "kg" + (kg_root / "data/merged").mkdir(parents=True) + census = {"source": {"manifest_sha256": digest, + "snapshot_path": str(kg_root / "data/merged/20000101")}} + census_path = tmp_path / "census.json" + census_path.write_text(json.dumps(census)) + census_hash = helper.file_hash(census_path) + recovery = {"state": "PRIVATE_STAGING_TRANSFER_AND_BYTE_READBACK_COMPLETE", + "census_manifest_sha256": census_hash, "source_archive": { + "extracted_directory": str(source), "manifest_sha256": digest, + "members": {n: {"bytes": x["bytes"], "sha256": x["sha256"]} + for n, x in manifest["members"].items()}}} + recovery_path = tmp_path / "recovery.json" + recovery_path.write_text(json.dumps(recovery)) + state = {"held": False, "released": False, "copies": 0, "status_calls": 0} + + def status(*args): + state["status_calls"] += 1 + return "changed" if failure == "owner_changed" and state["status_calls"] > 1 else "same" + + repo = SimpleNamespace(git=SimpleNamespace(status=status, check_ignore=lambda path: path), + head=SimpleNamespace(commit=SimpleNamespace(hexsha="owner-head"))) + + class Target: + def __init__(self, *args): + pass + + def open(self): + return repo + + class Manager: + @contextmanager + def lock(self, *args, **kwargs): + if failure == "lock_refusal": + raise RuntimeError("injected lease refusal") + state["held"] = True + try: + yield + finally: + state["held"] = False + state["released"] = True + + def fail_copy(*args, **kwargs): + assert state["held"] + state["copies"] += 1 + raise OSError("injected copy failure") + + monkeypatch.setattr(helper, "restore_absent", fail_copy) + monkeypatch.setattr(helper, "CENSUS_SHA", census_hash) + monkeypatch.setattr(helper, "SOURCE_SHA", digest) + monkeypatch.setattr(helper, "_snapshot", lambda *args: ( + (source / "manifest.json").read_bytes(), manifest, "merged-kg_edges.tsv")) + monkeypatch.setattr(helper, "_verify_source", lambda *args: "verified-source-hash") + monkeypatch.setattr(helper.shutil, "disk_usage", lambda path: SimpleNamespace(free=20 * 1024**3)) + monkeypatch.setitem(sys.modules, "kg_microbe_fleet.roots", SimpleNamespace( + resolve_kg_microbe_root=lambda **kwargs: kg_root)) + monkeypatch.setitem(sys.modules, "plugins.lock_manager", SimpleNamespace(LockManager=Manager)) + monkeypatch.setitem(sys.modules, "plugins.repository_settings", SimpleNamespace( + RepositoryTarget=Target, merged_repository_environment=lambda path: {}, + RepositorySettings=SimpleNamespace(from_environment=lambda **kwargs: SimpleNamespace( + get_target=lambda key: SimpleNamespace(path=tmp_path / "edge"))))) + results = tmp_path / "results" + monkeypatch.setattr(sys, "argv", ["restore", "--claw-root", str(tmp_path), + "--recovery-receipt", str(recovery_path), "--census", str(census_path), + "--results", str(results), "--apply"]) + with pytest.raises((RuntimeError, ValueError, OSError)): + helper.main() + assert state["copies"] == (1 if failure == "copy_failure" else 0) + assert not state["held"] + assert state["released"] == (failure != "lock_refusal") + assert json.loads((results / "failure.json").read_text())["status"] == "failed"