diff --git a/.github/uv-bootstrap.txt b/.github/uv-bootstrap.txt index 1d4369f..45d917f 100644 --- a/.github/uv-bootstrap.txt +++ b/.github/uv-bootstrap.txt @@ -1,2 +1,2 @@ # Verified bootstrap wheel used consistently by CI. -uv==0.11.21 --hash=sha256:b9ecdefa81db7e966d1655988cad6f840316228381dd69131ebc4ae9362bbccd --hash=sha256:00193e4e077c27ee3d66da356744dbf0b3aa59356dfbd9a9efb1dc8469af8ad7 +uv==0.13.4 --hash=sha256:05e4332791b4d55b6dde8695b7fa6079b26991519f44760887c5dfbd5ca4fe53 --hash=sha256:7b3cdf868e5415631fe797b648c0d6945d38d388f23367bdd0b6332e80a1fcb0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40f6801..fb50d74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,8 +137,8 @@ jobs: - name: Install uv env: - UV_VERSION: "0.11.21" - UV_SHA256: "b9ecdefa81db7e966d1655988cad6f840316228381dd69131ebc4ae9362bbccd" + UV_VERSION: "0.13.4" + UV_SHA256: "05e4332791b4d55b6dde8695b7fa6079b26991519f44760887c5dfbd5ca4fe53" run: | set -euo pipefail printf 'uv==%s --hash=sha256:%s\n' "$UV_VERSION" "$UV_SHA256" \ diff --git a/.github/workflows/release-notes.yml b/.github/workflows/release-notes.yml index 55367d4..b771451 100644 --- a/.github/workflows/release-notes.yml +++ b/.github/workflows/release-notes.yml @@ -44,8 +44,8 @@ jobs: - name: Install pinned uv run: | set -euo pipefail - printf 'uv==0.11.21 --hash=sha256:%s\n' \ - 'b9ecdefa81db7e966d1655988cad6f840316228381dd69131ebc4ae9362bbccd' \ + printf 'uv==0.13.4 --hash=sha256:%s\n' \ + '05e4332791b4d55b6dde8695b7fa6079b26991519f44760887c5dfbd5ca4fe53' \ > "$RUNNER_TEMP/uv-requirements.txt" python -m pip install --disable-pip-version-check --require-hashes \ --requirement "$RUNNER_TEMP/uv-requirements.txt" @@ -129,25 +129,25 @@ jobs: runner: ubuntu-24.04 container: ubuntu:24.04@sha256:534baea6a22c03a63003dbc8dbe78fe34bc0d7e595d9a9dc9834884ff530eb55 restic_archive_sha256: f415415624dcc452f2a02b8c33641791a8c6d6d3b65bbb3543fcf9a25151585c - uv_sha256: b9ecdefa81db7e966d1655988cad6f840316228381dd69131ebc4ae9362bbccd + uv_sha256: 05e4332791b4d55b6dde8695b7fa6079b26991519f44760887c5dfbd5ca4fe53 - release: "24.04" architecture: arm64 runner: ubuntu-24.04-arm container: ubuntu:24.04@sha256:534baea6a22c03a63003dbc8dbe78fe34bc0d7e595d9a9dc9834884ff530eb55 restic_archive_sha256: a5f64aaab53d51e311fa3829124c5b703f2d14cf187d8640b6be3b2b49376465 - uv_sha256: 00193e4e077c27ee3d66da356744dbf0b3aa59356dfbd9a9efb1dc8469af8ad7 + uv_sha256: 7b3cdf868e5415631fe797b648c0d6945d38d388f23367bdd0b6332e80a1fcb0 - release: "26.04" architecture: amd64 runner: ubuntu-24.04 container: ubuntu:26.04@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7 restic_archive_sha256: f415415624dcc452f2a02b8c33641791a8c6d6d3b65bbb3543fcf9a25151585c - uv_sha256: b9ecdefa81db7e966d1655988cad6f840316228381dd69131ebc4ae9362bbccd + uv_sha256: 05e4332791b4d55b6dde8695b7fa6079b26991519f44760887c5dfbd5ca4fe53 - release: "26.04" architecture: arm64 runner: ubuntu-24.04-arm container: ubuntu:26.04@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7 restic_archive_sha256: a5f64aaab53d51e311fa3829124c5b703f2d14cf187d8640b6be3b2b49376465 - uv_sha256: 00193e4e077c27ee3d66da356744dbf0b3aa59356dfbd9a9efb1dc8469af8ad7 + uv_sha256: 7b3cdf868e5415631fe797b648c0d6945d38d388f23367bdd0b6332e80a1fcb0 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -175,7 +175,7 @@ jobs: esac mkdir -p /opt/bbackup-build-tools /usr/bin/python3 -m venv /opt/bbackup-build-tools/uv - printf 'uv==0.11.21 --hash=sha256:%s\n' '${{ matrix.uv_sha256 }}' \ + printf 'uv==0.13.4 --hash=sha256:%s\n' '${{ matrix.uv_sha256 }}' \ > "$RUNNER_TEMP/uv-requirements.txt" /opt/bbackup-build-tools/uv/bin/python -m pip install \ --disable-pip-version-check --require-hashes \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a367b7..88d36f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,136 @@ current uv-based install, development, and release instructions. ## [Unreleased] +## [2.0.4] - 2026-10-21 + +### Added + +- Experimental restricted B2 source-key metadata assessment and independently + checked, separately signed central recovery acceptance. Live B2 protection + and complete-machine recovery remain unqualified. +- Experimental HTTPS S3 remote-primary capture with a dedicated bounded staging + filesystem, disabled cache, shared staging lease, pressure checks and + structured resource failures. A small live B2 workstation source trial is + recorded below; final remote-server qualification is deferred. +- Experimental central noncurrent-version retention with signed checkpoint + dependencies, explicit catalog and grants, shared repository leases, lock and + expiry checks, and journaled exact-version deletion without automatic replay. + Schema-2 B2 policies can select a native lock-metadata reader; schema-1 + generic S3 policies remain unchanged. Current-pack compaction and complete + provider protection qualification remain outstanding. +- Experimental central snapshot maintenance with explicit accepted-record + enrollment, policy-selected full snapshot IDs, separate Restic credentials, + bounded staging and `prune --max-repack-size 0`. Protected checkpoints and + unknown snapshots remain preserved; historical-version deletion stays + separate. Repacking and aged live reclamation remain unqualified. +- Opt-in reviewed Linux file and transactional SQLite coverage with private + inventory, exact scope and exclusions, drift checks, reviewed database + identity/metadata and reconstruction evidence, and root system unit rendering. + SQLite exports carry provenance in the encrypted capture manifest; live + database/journal files are omitted from file-tree capture. Complete-machine + recovery remains unqualified. +- Experimental manual PostgreSQL covered capture adds schema-2 private cluster + and database decisions, per-database `pg_dump` archives, digest-bound recovery + artifacts, and an explicit administrator-coordinated quiet-window assertion. + Database-level ownership, ACLs, and connection limits require a verified + reconstruction procedure after native restore. + A local covered capture and replacement rehearsal passed; stable v2.0.4 + scope remains the v2.0.3 tested core scope, and ordinary PostgreSQL adapters + are unchanged. +- Refreshed matching 1280×640 GitHub social preview artwork, embedded ownership metadata, checked provenance, and explicit repository-settings upload instructions. + +### Qualification evidence + +- On 2026-10-21, a local S3 emulator captured a small, compressible + 2,245,463-byte logical fixture with 1,769,472 bytes of combined writable + filesystem capacity. A controlled pressure failure stopped the next capture; + a separate read-only check and full restore verified the earlier snapshot. + The original harness stopped at a symlink-listing assertion; its failed + result is preserved separately from the successful recovery continuation. +- On 2026-10-21, live B2 central maintenance completed a zero-forget plan and + native no-repack prune. Both accepted snapshots restored afterward, all 32 + previously observed versions survived, and the catalog remained unchanged. + A separate changed-plan test rejected stale apply before maintenance-ledger + entry, forget, or prune; inspection created and released two temporary locks. + This small workstation result leaves aged reclamation, the protection window, + and off-host recovery unqualified. +- A separate live B2 maintenance interruption drill used one zero-forget plan + and one small fixture. After confirming the owned native prune process, its + durable intent, and a fresh lock witness, one SIGINT reached the owned CLI. + The native process stopped and the CLI exited 3. One read-only reconciliation + after process quiescence recorded an abandoned plan while prune stayed uncertain; apply was + not replayed. A no-lock read and two restores matched the fixture, all 69 + previously observed versions and markers survived, and the catalog was + unchanged. This does not claim prune completion or qualify provider protection. +- A separate new, empty File Lock-enabled B2 bucket tested overlapping native + lifecycle rules at creation. Read-only S3 inspection returned the expected + hide, orphan-marker, and noncurrent-delete actions, and central retention + refused the enabled expiration rules. No objects were uploaded or deleted; + this did not test elapsed expiry or physical reclamation. A read-only first- + entry inventory observation found no versions. The v2.0.4 stable scope + remains the v2.0.3 tested core scope, and requests #23–#26 remain open. +- On 2026-10-21, isolated PostgreSQL 17.11 containers passed covered capture + and restore into a fresh replacement database. The encrypted snapshot omitted + raw PostgreSQL files; recovered materials and their tested procedure rebuilt + database ownership, ACLs, connection limits, and three UTF-8 rows. The fixture + contained 10,006 logical bytes. This is local manual-path evidence, with no + off-host, unattended, or whole-machine qualification. +- On 2026-10-21, a root-covered files/SQLite rehearsal passed in two fresh + Ubuntu 24.04 AMD64 guests. The replacement restored encrypted data, rebuilt + fixture accounts and metadata, and verified application startup. The logical + fixture was 56,293 bytes; the two-guest run took 90.1 seconds. This is local + emulation, with no off-host custody or complete-machine qualification. +- On 2026-10-20, a small live B2 S3 remote-primary source trial passed two + captures, a repository check, restores of both snapshots, and seven denied + forbidden-operation checks using a bucket- and machine-prefix-restricted + source key. A separate verifier also reconstructed and checked historical + versions after simulated source compromise and restored the older snapshot. + The fixture, limits, and open qualification gaps are + documented in + [version 2 development](docs/development/version-2.md#live-backblaze-b2-source-trial). + This workstation result does not qualify B2 protection or production recovery; + AWS has no live result and the v2.0.4 stable support scope remains the + v2.0.3 tested core scope. +- A live central-retention rehearsal planned with a fresh thirty-day grace + period, ran a guarded zero-apply, and reconciled an empty plan. It identified + 10 protected versions in a 31-version inventory, which remained unchanged; + no S3 delete was sent. Empty reconciliation made no provider reads. The + native B2 metadata reader separately verified exact historical versions and + fresh governance-locked and legally held canaries. The routine retention key + was refused permanent deletion of both protected canaries, with survival + confirmed independently; both canaries were subsequently cleaned up. The + production CLI also passed schema-3 enrollment and schema-2 plan, zero-apply + and empty reconciliation against the existing catalog with its first-seen + times preserved. Aged deletion and uncertain live mutations remain unqualified. +- A fresh tiny B2 fixture measured initial/incremental capture interface TX of + 221,039/22,065 bytes and stored-version growth of 199,080/2,457 bytes. Both + captures, the full check and both restores passed within 1 MiB staging and + zero cache. These measurements include protocol/version overhead and do not + establish provider billing, large-scale performance or production savings. + ### Fixed +- Updated the pinned CI and release-build `uv` tool to 0.13.4 for + [GHSA-hq85-5cpj-mr62](https://github.com/advisories/GHSA-hq85-5cpj-mr62); + the amd64 and arm64 wheel digests are recorded in release dependency evidence. +- Updated `certifi` to 2026.10.19 to address + [GHSA-6q48-6c7w-9gw2](https://github.com/advisories/GHSA-6q48-6c7w-9gw2). +- Service operations disable Restic's cache when the child environment has no + cache location, so verified restore also works from minimal system services. +- Remote capture reconciliation uses Restic reads without cloud lock writes, + requires an acknowledged quiet window and refuses observed repository locks + before and after inspection. It never retries an uncertain upload. +- Remote database captures retain completed exports after failed or uncertain + uploads and receipt failures. Read-only reconciliation verifies the matching + snapshot before cleanup; bounded discovery and digest-bound explicit disposal + replace manual edits to retained staging. Reboot survival requires persistent + staging rather than `tmpfs`. +- Checkpoint receipts accept an absent expiry date instead of failing after the + encrypted kit is written. Known legacy receipt schemas migrate with a private + backup and preserved rows; additional schema constraints fail closed. +- Covered user/root timers preserve the reviewed temporary-directory source + view instead of substituting systemd private mounts; ordinary jobs retain + their private temporary directories. - Release publication locates the exact draft by release ID and refuses an existing draft or published tag before creation, including drafts hidden from lookup by tag. --- @@ -525,7 +653,8 @@ are still under development. See [implementation status](docs/development/versio The `Unreleased` comparison is a live `HEAD` link for navigation, not an immutable historical citation. -[Unreleased]: https://github.com/CruxExperts/BestBackup/compare/v2.0.3...HEAD +[Unreleased]: https://github.com/CruxExperts/BestBackup/compare/v2.0.4...HEAD +[2.0.4]: https://github.com/CruxExperts/BestBackup/compare/v2.0.3...v2.0.4 [2.0.3]: https://github.com/CruxExperts/BestBackup/compare/v2.0.2...v2.0.3 [2.0.2]: https://github.com/CruxExperts/BestBackup/compare/v2.0.1...v2.0.2 [2.0.1]: https://github.com/CruxExperts/BestBackup/compare/v2.0.0...v2.0.1 diff --git a/INSTALL.md b/INSTALL.md index 7e03ca0..a8c88e0 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,6 +1,6 @@ # Install bbackup -This guide covers stable v2.0.3 for its tested file, database, dashboard, and AMD64 package scope. B2/AWS, native Docker, ARM64 package functionality, and broader recovery guarantees remain unqualified; the retained 1.x tag remains available as v1.8.6. +This guide covers stable v2.0.4 for its tested file, database, dashboard, and AMD64 package scope. New cloud and Linux recovery workflows are experimental and unqualified; B2/AWS, native Docker, ARM64 package functionality, and broader recovery guarantees remain outside the qualified scope. The retained 1.x tag remains available as v1.8.6. ## Version 2 stable release @@ -9,12 +9,12 @@ The stable release uses Python 3.12 or newer and the `restic` executable. Instal Install the pinned stable release with uv: ```bash -uv tool install --force 'git+https://github.com/CruxExperts/BestBackup.git@v2.0.3' +uv tool install --force 'git+https://github.com/CruxExperts/BestBackup.git@v2.0.4' bbackup --version bbackup production --help ``` -The command is pinned to the `v2.0.3` source tag. To inspect or modify the source, use the development checkout below. +The command is pinned to the `v2.0.4` source tag. To inspect or modify the source, use the development checkout below. ### Optional native database clients @@ -22,11 +22,11 @@ PostgreSQL capture requires `pg_dump` and `psql` for source metadata; restoring ### Optional systemd scheduling -Version 2.0.3 supports explicit user-level installation through `service install`, with activation only when `--activate` is supplied. Scheduled file capture/copy/restore, missed-timer catch-up, and abrupt interruption without replay passed isolated Ubuntu 24.04/26.04 AMD64 guest drills; this does not qualify every host configuration or the root-owned Debian templates. Review [the quick start](QUICKSTART.md#apply-a-schedule-deliberately) and the [development candidate reboot qualification](docs/development/version-2.md#development-candidate-rebootable-systemd-qualification) before enabling a schedule. A user timer needs a user systemd manager and may need administrator-managed lingering to run after logout or at boot. +Version 2.0.4 supports explicit user-level installation through `service install`, with activation only when `--activate` is supplied. Scheduled file capture/copy/restore, missed-timer catch-up, and abrupt interruption without replay passed isolated Ubuntu 24.04/26.04 AMD64 guest drills; this does not qualify every host configuration or the root-owned Debian templates. Review [the quick start](QUICKSTART.md#apply-a-schedule-deliberately) and the [development candidate reboot qualification](docs/development/version-2.md#development-candidate-rebootable-systemd-qualification) before enabling a schedule. A user timer needs a user systemd manager and may need administrator-managed lingering to run after logout or at boot. ### Cloud repositories -Restic repositories can use local storage, Backblaze B2 through the S3 API, or Amazon S3. Cloud providers are configurable but not production-qualified in v2.0.3. Configure credentials in the supervised process environment. Do not place credentials in portable JSON policy or a repository URL. See [cloud storage](docs/cloud-storage.md) for experimental provider setup and limits. +Restic repositories can use local storage, Backblaze B2 through the S3 API, or Amazon S3. Cloud providers are configurable but not production-qualified in v2.0.4. Configure credentials in the supervised process environment. Do not place credentials in portable JSON policy or a repository URL. See [cloud storage](docs/cloud-storage.md) for experimental provider setup and limits. ## Existing 1.x release @@ -81,7 +81,7 @@ sudo env \ UV_TOOL_DIR=/opt/uv/tools \ UV_TOOL_BIN_DIR=/usr/local/bin \ "$UV" tool install --force \ - 'git+https://github.com/CruxExperts/BestBackup.git@v2.0.3' + 'git+https://github.com/CruxExperts/BestBackup.git@v2.0.4' bbackup --version ``` @@ -113,12 +113,12 @@ Install a newer release tag with `uv tool install --force` and the exact tag. Pr ## Debian package installation -Native Ubuntu packages target Ubuntu 24.04 with Python 3.12 and Ubuntu 26.04 with Python 3.14, on amd64 or arm64. The v2.0.3 production-qualified package scope is amd64 only; ARM64 artifacts, if built, are unqualified pending functional capture/restore smoke tests. Use a package built natively for the target release, architecture, and Python version; the package builder refuses cross-build labels. +Native Ubuntu packages target Ubuntu 24.04 with Python 3.12 and Ubuntu 26.04 with Python 3.14, on amd64 or arm64. The v2.0.4 production-qualified package scope is amd64 only; ARM64 artifacts, if built, are unqualified pending functional capture/restore smoke tests. Use a package built natively for the target release, architecture, and Python version; the package builder refuses cross-build labels. Inspect the package, contents, manifest, and checksum before installing. For the Ubuntu 24.04 AMD64 artifact: ```bash -PACKAGE=dist/deb/bbackup_2.0.3_ubuntu24.04_amd64.deb +PACKAGE=dist/deb/bbackup_2.0.4_ubuntu24.04_amd64.deb (cd "$(dirname "$PACKAGE")" && sha256sum -c "$(basename "${PACKAGE%.deb}.sha256")") dpkg-deb --info "$PACKAGE" dpkg-deb --contents "$PACKAGE" @@ -130,13 +130,13 @@ cat "$REVIEW_DIR/usr/share/doc/bbackup/build-manifest.json" Install only after reviewing that output: ```bash -sudo apt install ./dist/deb/bbackup_2.0.3_ubuntu24.04_amd64.deb +sudo apt install ./dist/deb/bbackup_2.0.4_ubuntu24.04_amd64.deb bbackup --version bbman --help ``` The release build matrix targets four uniquely named native packages: `bbackup__ubuntu24.04_amd64.deb`, `bbackup__ubuntu24.04_arm64.deb`, `bbackup__ubuntu26.04_amd64.deb`, and `bbackup__ubuntu26.04_arm64.deb`. A configured matrix is not evidence that all four artifacts have been built or qualified. Each package contains the versioned Python package, its locked dependencies, and the verified restic binary. Installation does not install Python packages into system Python, access a package index at runtime, enable or start services, or remove configuration, private bindings, credentials, repositories, or backup state. -The v2.0.3 release workflow also builds ARM64 packages, but its install smoke does not qualify backup/restore functionality on ARM64. Treat those packages as experimental. +The v2.0.4 release workflow also builds ARM64 packages, but its install smoke does not qualify backup/restore functionality on ARM64. Treat those packages as experimental. Package upgrades and removals refuse while either command is active. A root-owned marker blocks both launchers while package files are changing. Successful configuration verifies the installed payload checksum manifest before clearing the marker. On abort, a marker requires the exclusive process lock and complete payload validation before removal. Without a marker, the abort hook takes a shared lock, verifies the payload and launchers, and returns without clearing anything; existing clients can continue while transition writers remain excluded. diff --git a/QUICKSTART.md b/QUICKSTART.md index 4d557bb..c2e4a14 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -1,6 +1,6 @@ # Your first backup and verified restore -This guide covers v2.0.3 and its schema-3 policy. For an older v2.0.2 installation, use its [versioned quick start](https://github.com/CruxExperts/BestBackup/blob/v2.0.2/QUICKSTART.md) until you migrate. Keep an independently retrievable copy of each repository password. +This guide covers v2.0.4 and its schema-3 policy. For an older v2.0.2 installation, use its [versioned quick start](https://github.com/CruxExperts/BestBackup/blob/v2.0.2/QUICKSTART.md) until you migrate. Keep an independently retrievable copy of each repository password. ## Start with the guided dashboard @@ -117,6 +117,15 @@ Units are installed under an absolute `$XDG_CONFIG_HOME/systemd/user`, or `~/.co Omit `--activate` to install disabled units. The command never enables lingering, installs a system-wide service, or elevates privileges. User timers require an available user systemd manager; boot operation may require administrator-managed lingering. A differing existing unit is preserved for review. [Installation](INSTALL.md#optional-systemd-scheduling) documents qualification limits. +The experimental [reviewed Linux coverage workflow](docs/development/version-2.md#reviewed-linux-coverage-experimental) +adds explicit private scope discovery, review and drift checks, plus an +experimental manual PostgreSQL covered-capture path. PostgreSQL coverage +requires an externally coordinated administrative window and a manual +confirmation flag; scheduled covered PostgreSQL jobs remain blocked. Its +`service render-system` command renders root-controlled system units for +administrator review; it never installs them. A complete scope check does not +qualify complete-machine disaster recovery. + Persistent timers can catch up after an idle interruption. An interrupted mutation remains uncertain and blocks automatic replay until explicit reconciliation. Retention remains a separate reviewed action; scheduling does not silently prune data. ## Existing Docker and YAML users diff --git a/README.md b/README.md index 86ceb61..2039324 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ bbackup is an encrypted, restic-based backup tool for Linux system administrators, homelab and self-hosted operators, and small infrastructure teams. It brings local file backups, selected database workflows, verified file restores, and observable scheduled jobs into one operator workflow. -[![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-3776ab?style=flat-square&logo=python&logoColor=white)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/license-MIT-22c55e?style=flat-square)](LICENSE) [![Version 2.0.3](https://img.shields.io/badge/version-2.0.3-22c55e?style=flat-square)](https://github.com/CruxExperts/BestBackup/releases/latest) +[![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-3776ab?style=flat-square&logo=python&logoColor=white)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/license-MIT-22c55e?style=flat-square)](LICENSE) [![Version 2.0.4](https://img.shields.io/badge/version-2.0.4-22c55e?style=flat-square)](https://github.com/CruxExperts/BestBackup/releases/latest) ![bbackup: encrypted snapshot layers, an additional copy, and a restored document](docs/assets/bbackup-banner-v2.png) @@ -22,14 +22,29 @@ Use bbackup when you want to know what a job captured, whether each repository c ![Real bbackup dashboard with disposable example data and illustrative capacity](docs/assets/bbackup-dashboard-preview.svg) -## v2.0.3 tested scope +## v2.0.4 tested scope The qualified scope covers local file capture and restore; live capture and restore for PostgreSQL 17.11, MySQL 8.4.12, and MariaDB 11.8.9; headless dashboard happy paths; and AMD64 Ubuntu 24.04/26.04 packages. A separate real-guest user-timer reboot drill covers only the components and conditions recorded in the [qualification evidence](docs/development/version-2.md#development-candidate-rebootable-systemd-qualification). B2/AWS, native Docker, ARM64 backup/restore, cross-application-version upgrades and rollback, root-owned systemd package templates, and cloud or off-host protection remain unqualified. ARM64 packages may be built, but build or package-install smoke checks alone do not qualify backup/restore. Keep an independently retrievable copy of repository passwords; a second repository on the same host does not protect against host loss. See [installation](INSTALL.md) for prerequisites and release install instructions. +Recent live B2 evidence covers a small interrupted maintenance run +whose prune outcome remained uncertain, and a separate lifecycle-overlap guard +that made central retention refuse the configured bucket. Neither test proves +prune completion, elapsed lifecycle expiry, physical reclamation, or provider +protection. These workflows ship in v2.0.4 as experimental and remain +unqualified. See the [maintenance interruption record](docs/development/version-2.md#live-b2-central-maintenance-interruption-trial) +and [lifecycle guard evidence](docs/cloud-storage.md#b2-lifecycle-overlap-guard-experimental). +Requests #23–#26 remain open; the qualified core scope is unchanged from v2.0.3. + ## Start with the guided workflow +Experimental restricted cloud credentials, central recovery and retention, +bounded remote-primary capture, and reviewed Linux coverage ship in v2.0.4 and +are [documented with their limits](docs/development/version-2.md#experimental-cloud-and-machine-recovery-features). +The reviewed Linux coverage work includes an experimental manual PostgreSQL +path. These features do not expand the tested stable scope. + For packaged releases, start with the [latest release downloads](https://github.com/CruxExperts/BestBackup/releases/latest) and [installation guide](INSTALL.md). From a development checkout with Python 3.12+, uv, and restic installed: diff --git a/VERSION b/VERSION index 50ffc5a..2165f8f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.3 +2.0.4 diff --git a/bbackup/__init__.py b/bbackup/__init__.py index 34d5928..665ff82 100644 --- a/bbackup/__init__.py +++ b/bbackup/__init__.py @@ -3,5 +3,5 @@ Encrypted restic-based workflows; legacy Docker commands remain available. """ -__version__ = "2.0.3" +__version__ = "2.0.4" __author__ = "Slavic Kozyuk / Crux Experts LLC" diff --git a/bbackup/b2.py b/bbackup/b2.py index fe0dd7b..e04a280 100644 --- a/bbackup/b2.py +++ b/bbackup/b2.py @@ -20,6 +20,7 @@ class B2Error(RuntimeError): _ADMIN = { "deleteFiles", + "listKeys", "deleteKeys", "writeKeys", "writeBuckets", @@ -34,9 +35,80 @@ class B2Error(RuntimeError): } _SAFE_CAPABILITIES = { "listBuckets", "listAllBucketNames", "listFiles", "readFiles", "writeFiles", - "readBucketEncryption", "readBucketRetentions", "readFileRetentions", - "readFileLegalHolds", "readBucketReplications", "readBucketNotifications", + "readBuckets", "readBucketEncryption", "readBucketRetentions", + "readFileRetentions", "readFileLegalHolds", "readBucketReplications", + "readBucketNotifications", } +_INSPECTION_CAPABILITIES = {"listBuckets", "listFiles", "readFiles"} +_PROFILE_CAPABILITIES = _INSPECTION_CAPABILITIES | {"writeFiles"} +_LOCK_READ_CAPABILITIES = { + "listBuckets", "readFiles", "readFileRetentions", "readFileLegalHolds", +} + + +def _restricted_source_profile( + *, bucket_id, bucket_name, allowed_buckets, prefix, allowed_prefix, capabilities +): + """Assess the observed key declaration without claiming live denial proof.""" + reasons = [] + bucket_bound = bool(allowed_buckets and len(allowed_buckets) == 1) + bucket_matches = bool( + bucket_bound and bucket_id == allowed_buckets[0]["id"] + and bucket_name == allowed_buckets[0]["name"] + ) + if not bucket_bound: + reasons.append("key_not_bucket_bound") + if not bucket_id: + reasons.append("selected_bucket_id_unavailable") + elif bucket_bound and not bucket_matches: + reasons.append("key_bucket_id_mismatch") + + prefix_restricted = bool(allowed_prefix) + prefix_canonical = bool( + prefix_restricted and allowed_prefix.endswith("/") and "\0" not in allowed_prefix + ) + prefix_contains_request = bool( + prefix_canonical and prefix.startswith(allowed_prefix) + ) + if not prefix_restricted: + reasons.append("machine_prefix_restriction_missing") + elif not prefix_canonical: + reasons.append("machine_prefix_not_canonical_directory") + elif not prefix_contains_request: + reasons.append("requested_prefix_outside_machine_prefix") + + observed = set(capabilities) + missing = sorted(_PROFILE_CAPABILITIES - observed) + if missing: + reasons.append("required_capabilities_missing") + if "deleteFiles" in observed: + reasons.append("permanent_deletion_capability_present") + if observed & (_ADMIN - {"deleteFiles"}): + reasons.append("administrative_capability_present") + if observed - _ADMIN - _SAFE_CAPABILITIES: + reasons.append("unrecognized_capability_present") + + return { + "version": 1, + "status": "eligible_from_observed_metadata" if not reasons else "unqualified", + "basis": "sdk_authorization_metadata_only", + "reasons": reasons, + "required_capabilities": sorted(_PROFILE_CAPABILITIES), + "observed_capabilities": sorted(observed), + "missing_required_capabilities": missing, + "scope": { + "key_bucket_bound": bucket_bound, + "key_bucket_matches_selected_bucket": bucket_matches, + "machine_prefix_restricted": prefix_restricted, + "machine_prefix_canonical_directory": prefix_canonical, + "requested_prefix_within_machine_prefix": prefix_contains_request, + }, + "negative_test_proof": { + "status": "not_performed", + "permanent_deletion_denial_demonstrated": False, + "administrative_denial_demonstrated": False, + }, + } def _endpoint(value, pattern): @@ -54,6 +126,137 @@ def _endpoint(value, pattern): return value.rstrip("/") +class B2VersionLockReader: + """Read one exact B2 native file version, retaining authorization flags. + + The application key is supplied explicitly and exists only in SDK memory. + Neither normalized SDK FileVersion objects nor S3's ambiguous 404 responses + can establish that an absent lock is safe for deletion. + """ + + def __init__( + self, bucket: str, bucket_id: str, prefix: str, endpoint_url: str, + *, access_key_id: str | None = None, secret_access_key: str | None = None, + api=None, + ) -> None: + if ( + not isinstance(bucket, str) or not re.fullmatch(r"[a-zA-Z0-9-]{6,63}", bucket) + or not isinstance(bucket_id, str) or not bucket_id or any(c in bucket_id for c in "\0\r\n") + or not isinstance(prefix, str) or not prefix or not prefix.endswith("/") + or prefix.startswith("/") or "\0" in prefix + ): + raise B2Error("Invalid B2 lock-reader target") + endpoint = _endpoint(endpoint_url, r"s3\.[a-z0-9-]+\.backblazeb2\.com") + if api is None: + if not access_key_id or not secret_access_key: + raise B2Error("Explicit B2 lock-reader credentials are required") + try: + from b2sdk.v3 import B2Api, InMemoryAccountInfo + + api = B2Api( + InMemoryAccountInfo(), max_upload_workers=1, + max_copy_workers=1, max_download_workers=1, + ) + api.authorize_account(access_key_id, secret_access_key) + except Exception: + raise B2Error("Could not authorize B2 lock reader") from None + try: + observed_endpoint = _endpoint( + api.account_info.get_s3_api_url(), r"s3\.[a-z0-9-]+\.backblazeb2\.com" + ) + _endpoint(api.account_info.get_api_url(), r"api\d+\.backblazeb2\.com") + allowed = api.account_info.get_allowed() + if not isinstance(allowed, dict): + raise ValueError() + capabilities = allowed.get("capabilities") + if not isinstance(capabilities, list) or not all(isinstance(c, str) for c in capabilities): + raise ValueError() + if not _LOCK_READ_CAPABILITIES <= set(capabilities): + raise ValueError() + allowed_buckets = allowed.get("buckets") + if allowed_buckets is not None and ( + not isinstance(allowed_buckets, list) + or not any( + isinstance(item, dict) and item.get("id") == bucket_id + and item.get("name") == bucket for item in allowed_buckets + ) + ): + raise ValueError() + allowed_prefix = allowed.get("namePrefix") + if allowed_prefix is not None and ( + not isinstance(allowed_prefix, str) or not prefix.startswith(allowed_prefix) + ): + raise ValueError() + buckets = api.list_buckets(bucket_name=bucket) + if ( + not isinstance(buckets, list) or len(buckets) != 1 + or buckets[0].name != bucket or buckets[0].id_ != bucket_id + or observed_endpoint != endpoint + ): + raise ValueError() + except Exception: + raise B2Error("B2 lock-reader endpoint, authority, or bucket differs from target") from None + self.bucket = bucket + self.bucket_id = bucket_id + self.prefix = prefix + self.endpoint_url = endpoint + self._api = api + self._credential_pair = (access_key_id, secret_access_key) if access_key_id and secret_access_key else None + + def matches_credentials(self, pair: tuple[str, str]) -> bool: + return self._credential_pair == pair + + def read(self, key: str, file_id: str, *, now) -> tuple[dict, bool]: + if ( + not isinstance(key, str) or not key.startswith(self.prefix) + or not isinstance(file_id, str) or not file_id or any(c in file_id for c in "\0\r\n") + ): + raise B2Error("B2 lock-reader version target is invalid") + try: + info = self._api.session.get_file_info_by_id(file_id) + if not isinstance(info, dict) or ( + info.get("bucketId") != self.bucket_id or info.get("fileId") != file_id + or info.get("fileName") != key or info.get("action") != "upload" + ): + raise ValueError() + retention = info.get("fileRetention") + legal = info.get("legalHold") + if ( + not isinstance(retention, dict) or not isinstance(legal, dict) + or retention.get("isClientAuthorizedToRead") is not True + or legal.get("isClientAuthorizedToRead") is not True + or "value" not in retention or "value" not in legal + ): + raise ValueError() + setting = retention["value"] + if not isinstance(setting, dict) or set(setting) != {"mode", "retainUntilTimestamp"}: + raise ValueError() + mode, until_ms = setting["mode"], setting["retainUntilTimestamp"] + if mode is None: + if until_ms is not None: + raise ValueError() + elif mode not in {"governance", "compliance"} or type(until_ms) is not int or until_ms < 0: + raise ValueError() + legal_value = legal["value"] + if legal_value not in ("on", "off", None): + raise ValueError() + from datetime import datetime, timezone + + until = datetime.fromtimestamp(until_ms / 1000, tz=timezone.utc) if until_ms is not None else None + if not isinstance(now, datetime) or now.tzinfo is None or now.utcoffset() is None: + raise ValueError() + evidence = { + "provider": "b2-native", "bucket": self.bucket, + "bucket_id": self.bucket_id, "key": key, "file_id": file_id, + "action": "upload", "legal_hold": "ON" if legal_value == "on" else "OFF", + "legal_hold_value": legal_value, + "mode": mode, "retain_until_timestamp_ms": until_ms, + } + return evidence, legal_value == "on" or (until is not None and until > now) + except Exception: + raise B2Error("B2 native per-version lock evidence is unknown") from None + + def inspect_sdk( bucket: str, *, @@ -87,18 +290,35 @@ def inspect_sdk( ) api.authorize_account(key_id, key) allowed = api.account_info.get_allowed() - capabilities = allowed["capabilities"] + if not isinstance(allowed, dict): + raise B2Error("B2 authorization metadata is invalid") + capabilities = allowed.get("capabilities") if ( not isinstance(capabilities, list) - or any(not isinstance(c, str) for c in capabilities) - or not {"listBuckets", "listFiles", "readFiles"}.issubset(capabilities) + or any(not isinstance(c, str) or not c for c in capabilities) ): + raise B2Error("B2 authorization metadata is invalid") + if not _INSPECTION_CAPABILITIES.issubset(capabilities): raise B2Error( "B2 key needs listBuckets, listFiles, and readFiles for inspection" ) - allowed_prefix = allowed.get("namePrefix") or "" - if not isinstance(allowed_prefix, str) or not prefix.startswith(allowed_prefix): - raise B2Error("Requested prefix is outside the key scope") + # b2sdk v3 normalizes provider authorization into a list of buckets. + # None means unrestricted; one matching id/name is the source profile. + allowed_buckets = allowed.get("buckets") + allowed_prefix = allowed.get("namePrefix") + if ( + allowed_buckets is not None + and (not isinstance(allowed_buckets, list) or any( + not isinstance(item, dict) + or not isinstance(item.get("id"), str) or not item["id"] + or not isinstance(item.get("name"), str) or not item["name"] + for item in allowed_buckets + )) + ) or ( + allowed_prefix is not None + and (not isinstance(allowed_prefix, str) or "\0" in allowed_prefix) + ): + raise B2Error("B2 authorization metadata is invalid") s3_url = _endpoint( api.account_info.get_s3_api_url(), r"s3\.[a-z0-9-]+\.backblazeb2\.com" ) @@ -106,15 +326,32 @@ def inspect_sdk( if len(matches) != 1 or matches[0].name != bucket: raise B2Error("B2 bucket is missing or inaccessible") selected = matches[0] - # Ask for one version to verify listing access without walking a bucket. - versions = iter( - selected.ls(path=prefix, latest_only=False, recursive=True, fetch_count=1) + selected_bucket_id = getattr(selected, "id_", None) + if selected_bucket_id is not None and not isinstance(selected_bucket_id, str): + raise B2Error("B2 bucket identity metadata is invalid") + profile = _restricted_source_profile( + bucket_id=selected_bucket_id, + bucket_name=bucket, + allowed_buckets=allowed_buckets, + prefix=prefix, + allowed_prefix=allowed_prefix, + capabilities=capabilities, ) - try: - next(versions, None) - finally: - if hasattr(versions, "close"): - versions.close() + version_listing_verified = False + # Inspection remains available to broader observer keys. The stricter + # machine-directory assessment is separate from the actual API scope. + if prefix.startswith(allowed_prefix or ""): + versions = iter( + selected.ls( + path=prefix, latest_only=False, recursive=True, fetch_count=1 + ) + ) + try: + next(versions, None) + finally: + if hasattr(versions, "close"): + versions.close() + version_listing_verified = True rules = selected.lifecycle_rules if not isinstance(rules, list): raise ValueError() @@ -146,7 +383,7 @@ def inspect_sdk( "s3_endpoint": s3_url, "prefix": prefix, "bucket_access_verified": True, - "version_listing_verified": True, + "version_listing_verified": version_listing_verified, "read_capability_present": True, "write_capability_present": "writeFiles" in capabilities, "credential_administration_denied": not administrative, @@ -155,6 +392,7 @@ def inspect_sdk( "observed_noncurrent_days": min(noncurrent_days) if noncurrent_days else None, + "restricted_source_profile": profile, "protection_qualified": False, "live_recovery_demonstrated": False, } diff --git a/bbackup/capture.py b/bbackup/capture.py index 9418b8d..efa8def 100644 --- a/bbackup/capture.py +++ b/bbackup/capture.py @@ -4,6 +4,8 @@ from collections.abc import Mapping from contextlib import ExitStack, contextmanager +import hashlib +import hmac import json import os import re @@ -12,9 +14,338 @@ import stat import tempfile import time +import uuid +from typing import Any from .models import DatabaseBinding, Source, strict_json +from .linux_coverage import CoverageError, verify_sqlite_identity from .operations import OperationPlan, OperationRunner, OperationStatus +from .resource_limits import ResourceLimitError, ResourceMonitor + + +class DurableCaptureArtifacts: + """Private, operation-owned remote exports; never discard uncertain data.""" + + def __init__(self, staging: Path, state: Path, repository_id: str, + token: str, job: str): + if not re.fullmatch(r"[a-f0-9]{64}", repository_id) or not re.fullmatch(r"[a-f0-9]{32}", token): + raise CaptureError("Invalid durable capture identity") + self.staging = Path(staging) + self.directory = self.staging / f"capture-{token}" + self.record_path = Path(state) / "capture-artifacts" / f"{token}.json" + self.record = {"schema_version": 1, "repository_id": repository_id, + "capture_token": token, "job": job, + "stage": str(self.directory), "stage_inode": None, + "phase": "prepared", "pending": [], "exports": [], "manifest": None} + + def _save(self): + parent = self.record_path.parent + parent.mkdir(mode=0o700, parents=True, exist_ok=True) + parent_info = parent.lstat() + if (not stat.S_ISDIR(parent_info.st_mode) or parent_info.st_uid != os.getuid() + or parent_info.st_mode & 0o077): + raise CaptureError("Private capture record directory is not private") + temporary = parent / f".{self.record_path.name}.{uuid.uuid4().hex}.tmp" + try: + with temporary.open("x", encoding="utf-8") as output: + temporary.chmod(0o600) + json.dump(self.record, output, sort_keys=True, separators=(",", ":")) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, self.record_path) + descriptor = os.open(parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + temporary.unlink(missing_ok=True) + + @staticmethod + def _sync_directory(path: Path): + descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + def prepare(self): + if self.record_path.exists() or self.directory.exists(): + raise CaptureError("Durable capture identity is already in use") + self._save() + self.directory.mkdir(mode=0o700) + self._sync_directory(self.staging) + self.record["stage_inode"] = self.directory.lstat().st_ino + self._save() + + def recover_empty_preparation(self): + """Bind the exact empty stage left between mkdir and inode persistence.""" + + if (self.record.get("phase") != "prepared" or self.record.get("stage_inode") is not None + or self.record.get("pending") or self.record.get("exports") + or self.record.get("manifest") is not None or not self.directory.exists()): + return + info = self.directory.lstat() + if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() + or info.st_mode & 0o077 or info.st_dev != self.staging.stat().st_dev + or any(self.directory.iterdir())): + raise CaptureError("Unrecorded capture stage is not an empty owned preparation") + self.record["stage_inode"] = info.st_ino + self._save() + + def begin_export(self, path: Path): + name = self._name(path) + if name in self.record["pending"] or any(item["name"] == name for item in self.record["exports"]): + raise CaptureError("Duplicate durable export path") + self.record["pending"].append(name) + self._save() + + def finish_export(self, path: Path): + name = self._name(path) + if name not in self.record["pending"]: + raise CaptureError("Durable export was not prepared") + self.record["exports"].append({"name": name, **self._fingerprint(path)}) + self._sync_directory(self.directory) + self.record["pending"].remove(name) + self._save() + + def ready(self, manifest: Path | None): + if manifest is not None: + name = self._name(manifest) + if name not in self.record["pending"]: + raise CaptureError("Durable capture manifest was not prepared") + self.record["manifest"] = {"name": name, **self._fingerprint(manifest)} + self.record["pending"].remove(name) + self._save() + if self.record["pending"]: + raise CaptureError("An export is incomplete") + self._sync_directory(self.directory) + if self.record["manifest"] is not None and self.record["manifest"]["size"] > 64 * 1024: + raise CaptureError("Durable capture manifest exceeds the inspection bound") + self.record["phase"] = "ready" + self._save() + + def _name(self, path: Path) -> str: + path = Path(path) + if path.parent != self.directory or path.name in ("", ".", ".."): + raise CaptureError("Durable export is outside its stage") + return path.name + + @staticmethod + def _fingerprint(path: Path) -> dict: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(descriptor) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or info.st_mode & 0o077 or info.st_nlink != 1): + raise CaptureError("Durable capture file is not an owned private regular file") + digest = hashlib.sha256() + with os.fdopen(descriptor, "rb", closefd=False) as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + after = os.fstat(descriptor) + if (after.st_size != info.st_size or after.st_mtime_ns != info.st_mtime_ns + or after.st_ctime_ns != info.st_ctime_ns): + raise CaptureError("Durable capture changed while hashing") + os.fsync(descriptor) + return {"size": info.st_size, "sha256": digest.hexdigest()} + finally: + os.close(descriptor) + + @classmethod + def load(cls, staging: Path, state: Path, repository_id: str, token: str): + artifact = cls(staging, state, repository_id, token, "") + parent_info = artifact.record_path.parent.lstat() + if (not stat.S_ISDIR(parent_info.st_mode) or parent_info.st_uid != os.getuid() + or parent_info.st_mode & 0o077): + raise CaptureError("Private capture record directory changed") + descriptor = os.open(artifact.record_path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(descriptor) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or info.st_mode & 0o077 or info.st_size > 1024 * 1024): + raise CaptureError("Durable capture record is not private") + with os.fdopen(descriptor, "r", encoding="utf-8", closefd=False) as stream: + record = json.load(stream) + finally: + os.close(descriptor) + if (not isinstance(record, dict) or record.get("schema_version") != 1 + or record.get("repository_id") != repository_id + or record.get("capture_token") != token + or record.get("stage") != str(artifact.directory)): + raise CaptureError("Durable capture record identity changed") + artifact.record = record + return artifact + + def record_sha256(self) -> str: + return self._fingerprint(self.record_path)["sha256"] + + def reviewed_failure(self, outcome: str): + if outcome not in ("failed", "abandoned"): + raise CaptureError("Only failed or abandoned captures retain exports") + self.validate() + observed = [] + for name in self.record["pending"]: + path = self.directory / name + observed.append({"name": name, **self._fingerprint(path)} if path.exists() + else {"name": name, "absent": True}) + self.record["pending_observed"] = observed + self.record["reviewed_outcome"] = outcome + self._save() + + def discard(self, expected_sha256: str): + if (not isinstance(expected_sha256, str) + or not re.fullmatch(r"[a-f0-9]{64}", expected_sha256) + or not hmac.compare_digest(self.record_sha256(), expected_sha256)): + raise CaptureError("Durable capture record digest changed") + self.validate() + if (self.record.get("reviewed_outcome") not in ("failed", "abandoned") + or self.record["phase"] not in ("prepared", "ready", "discard_started")): + raise CaptureError("Durable capture is not an explicitly reviewed failure") + self.record["phase"] = "discard_started" + self._save() + if self.directory.exists(): + completed = [*self.record["exports"]] + if self.record["manifest"] is not None: + completed.append(self.record["manifest"]) + for item in completed: + path = self.directory / item["name"] + if path.exists(): + if self._fingerprint(path) != {"size": item["size"], "sha256": item["sha256"]}: + raise CaptureError("Completed durable export changed during disposal") + path.unlink() + observed = {item["name"]: item for item in self.record["pending_observed"]} + for name in self.record["pending"]: + path = self.directory / name + if path.exists(): + item = observed[name] + if item.get("absent") or self._fingerprint(path) != { + "size": item["size"], "sha256": item["sha256"]}: + raise CaptureError("Pending durable export changed during disposal") + path.unlink() + self._sync_directory(self.directory) + self.directory.rmdir() + self._sync_directory(self.staging) + self.record["phase"] = "discarded" + self._save() + + def validate(self): + record = self.record + if (record.get("schema_version") != 1 or record.get("phase") not in + ("prepared", "ready", "cleanup_started", "cleaned", "discard_started", "discarded") + or not isinstance(record.get("job"), str) or not record["job"] + or not isinstance(record.get("pending"), list) + or not isinstance(record.get("exports"), list) + or record.get("stage_inode") is not None and + (type(record["stage_inode"]) is not int or record["stage_inode"] <= 0)): + raise CaptureError("Durable capture record is invalid") + completed = [*record["exports"]] + if record.get("manifest") is not None: + completed.append(record["manifest"]) + names = [] + for item in completed: + if (not isinstance(item, dict) or set(item) != {"name", "size", "sha256"} + or type(item["size"]) is not int or item["size"] < 0 + or not isinstance(item["sha256"], str) + or not re.fullmatch(r"[a-f0-9]{64}", item["sha256"])): + raise CaptureError("Durable capture file record is invalid") + names.append(item["name"]) + names.extend(record["pending"]) + if (any(not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", name) + or name in (".", "..") for name in names) or len(set(names)) != len(names)): + raise CaptureError("Durable capture file names are invalid") + if record["phase"] in ("ready", "cleanup_started", "cleaned") and record["exports"] and record.get("manifest") is None: + raise CaptureError("Durable capture manifest is missing") + observed = record.get("pending_observed") + if record.get("reviewed_outcome") in ("failed", "abandoned"): + if not isinstance(observed, list) or len(observed) != len(record["pending"]): + raise CaptureError("Reviewed pending export record is invalid") + for item in observed: + if (not isinstance(item, dict) or item.get("name") not in record["pending"] + or (set(item) != {"name", "absent"} and + (set(item) != {"name", "size", "sha256"} + or type(item.get("size")) is not int or item["size"] < 0 + or not isinstance(item.get("sha256"), str) + or not re.fullmatch(r"[a-f0-9]{64}", item["sha256"])))): + raise CaptureError("Reviewed pending export record is invalid") + if len({item["name"] for item in observed}) != len(observed): + raise CaptureError("Reviewed pending export record is duplicated") + elif observed is not None: + raise CaptureError("Pending export observations lack a reviewed failure") + directory = self.directory + if (not directory.exists() and self.record["phase"] == "prepared" + and self.record["stage_inode"] is None + and not self.record["exports"] and not self.record["pending"]): + return + if not directory.exists() and self.record["phase"] in ( + "cleanup_started", "cleaned", "discard_started", "discarded"): + return + info = directory.lstat() + if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() + or info.st_mode & 0o077 or info.st_dev != self.staging.stat().st_dev + or info.st_ino != self.record["stage_inode"]): + raise CaptureError("Durable capture stage changed") + entries = {item.name: item for item in directory.iterdir()} + expected = set(self.record["pending"]) + expected.update(item["name"] for item in completed) + if not set(entries).issubset(expected): + raise CaptureError("Durable capture stage contains unknown files") + for name, path in entries.items(): + if name in ("", ".", "..") or "/" in name: + raise CaptureError("Invalid durable capture stage entry") + info = path.lstat() + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or info.st_mode & 0o077 or info.st_nlink != 1 or info.st_dev != directory.stat().st_dev): + raise CaptureError("Durable capture stage contains an unsafe file") + missing_allowed = self.record["phase"] in ( + "cleanup_started", "cleaned", "discard_started", "discarded") + for item in completed: + name = item["name"] + if name not in entries: + if missing_allowed: + continue + raise CaptureError("Completed durable export is missing") + if self._fingerprint(entries[name]) != {"size": item["size"], "sha256": item["sha256"]}: + raise CaptureError("Completed durable export changed") + if observed is not None: + for item in observed: + name = item["name"] + if item.get("absent") is True: + if name in entries: + raise CaptureError("Reviewed pending export appeared after inspection") + elif name not in entries: + if self.record["phase"] not in ("discard_started", "discarded"): + raise CaptureError("Reviewed pending export is missing") + elif self._fingerprint(entries[name]) != { + "size": item["size"], "sha256": item["sha256"]}: + raise CaptureError("Reviewed pending export changed") + if self.record["phase"] in ("ready", "cleanup_started", "cleaned") and self.record["pending"]: + raise CaptureError("Durable capture record is inconsistent") + + def cleanup_success(self): + self.validate() + if self.record["phase"] not in ("ready", "cleanup_started", "cleaned"): + raise CaptureError("Durable capture was not ready for cleanup") + if not self.directory.exists(): + if self.record["phase"] != "cleaned": + self.record["phase"] = "cleaned" + self._save() + return + self.record["phase"] = "cleanup_started" + self._save() + completed = [*self.record["exports"]] + if self.record["manifest"] is not None: + completed.append(self.record["manifest"]) + for item in completed: + path = self.directory / item["name"] + if path.exists(): + if self._fingerprint(path) != {"size": item["size"], "sha256": item["sha256"]}: + raise CaptureError("Completed durable export changed during cleanup") + path.unlink() + self.directory.rmdir() + self._sync_directory(self.staging) + self.record["phase"] = "cleaned" + self._save() class CaptureError(RuntimeError): @@ -75,6 +406,7 @@ def _run_native( timeout: float, cancel, on_stdout=None, + monitor: ResourceMonitor | None = None, ): plan = OperationPlan.create( f"capture:{source.connection}:{source.name}", @@ -84,7 +416,12 @@ def _run_native( timeout_seconds=timeout, ) try: - result = runner.run(plan, on_stdout=on_stdout, cancel_event=cancel) + options = {"on_stdout": on_stdout, "cancel_event": cancel} + if monitor is not None: + options["on_tick"] = monitor.check + result = runner.run(plan, **options) + except ResourceLimitError: + raise except Exception: raise CaptureError("Native database capture command failed") from None if result.status is not OperationStatus.SUCCEEDED or result.exit_code != 0: @@ -94,9 +431,26 @@ def _run_native( return result -def _capture_postgresql(source, binding, target, runner, deadline, cancel): +def _bounded_write(output, chunk: bytes, monitor: ResourceMonitor | None): + if monitor is not None: + monitor.check(len(chunk)) + return output.write(chunk) + + +def _file_sha256(path: Path, monitor: ResourceMonitor | None = None) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + if monitor is not None: + monitor.check() + digest.update(chunk) + return digest.hexdigest() + + +def _capture_postgresql(source, binding, target, runner, deadline, cancel, monitor=None): assert binding.service_file is not None and binding.service_name is not None - env_values = {"PGSERVICEFILE": str(binding.service_file), "PGPASSFILE": "/dev/null"} + env_values = {"PGSERVICEFILE": str(binding.service_file), "PGPASSFILE": "/dev/null", + "TMPDIR": str(target.parent)} if binding.password_file is not None: env_values["PGPASSFILE"] = str(binding.password_file) env = _native_environment(**env_values) @@ -120,6 +474,7 @@ def _capture_postgresql(source, binding, target, runner, deadline, cancel): env=env, timeout=max(0.001, deadline - time.monotonic()), cancel=cancel, + monitor=monitor, ) try: if inspected.stdout_bytes != len(inspected.stdout_tail): @@ -157,7 +512,8 @@ def _capture_postgresql(source, binding, target, runner, deadline, cancel): env=env, timeout=max(0.001, deadline - time.monotonic()), cancel=cancel, - on_stdout=output.write, + on_stdout=lambda chunk: _bounded_write(output, chunk, monitor), + monitor=monitor, ) output.flush() os.fsync(output.fileno()) @@ -173,12 +529,12 @@ def _capture_postgresql(source, binding, target, runner, deadline, cancel): return options -def _capture_mysql(source, binding, target, runner, deadline, cancel): +def _capture_mysql(source, binding, target, runner, deadline, cancel, monitor=None): assert binding.options_file is not None and source.database is not None options = f"--defaults-file={binding.options_file}" client = "mysql" if source.kind == "mysql" else "mariadb" login_path_option = ["--no-login-paths"] if source.kind == "mysql" else [] - env_values = {} + env_values = {"TMPDIR": str(target.parent)} if source.kind == "mariadb": # MariaDB clients may read ~/.mylogin.cnf; confine HOME to this private # staging directory and explicitly disable the login-path file. @@ -206,6 +562,7 @@ def _capture_mysql(source, binding, target, runner, deadline, cancel): env=env, timeout=max(0.001, deadline - time.monotonic()), cancel=cancel, + monitor=monitor, ) if inspected.stdout_bytes != len(inspected.stdout_tail): raise CaptureError("MySQL and MariaDB database metadata is invalid") @@ -258,6 +615,7 @@ def inspect_engine(chunk: bytes) -> None: timeout=max(0.001, deadline - time.monotonic()), cancel=cancel, on_stdout=inspect_engine, + monitor=monitor, ) if non_innodb: raise CaptureError("MySQL and MariaDB capture requires InnoDB tables") @@ -286,7 +644,8 @@ def inspect_engine(chunk: bytes) -> None: env=env, timeout=max(0.001, deadline - time.monotonic()), cancel=cancel, - on_stdout=output.write, + on_stdout=lambda chunk: _bounded_write(output, chunk, monitor), + monitor=monitor, ) output.flush() os.fsync(output.fileno()) @@ -315,11 +674,17 @@ def capture_sources( protected_paths: tuple[Path, ...] = (), operation_id: str | None = None, runner: OperationRunner | None = None, + resource_monitor: ResourceMonitor | None = None, + coverage_sqlite: Mapping[str, Mapping[str, Any]] | None = None, + coverage_postgresql: Mapping[str, Mapping[str, Any]] | None = None, + validate_coverage=None, + durable_artifacts: DurableCaptureArtifacts | None = None, ): """Yield source paths plus temporary native exports and private Docker metadata. Ordinary file trees are live, best-effort captures, not atomic filesystem - snapshots. PostgreSQL and SQLite use transactional exports. MySQL and + snapshots. Each PostgreSQL and SQLite export is transactional within that + database; separate files and databases do not share a transaction. MySQL and MariaDB require an explicit read-lock quiescence declaration and use a server read lock during export. Docker sources require a local daemon and remain stopped only while the caller runs the restic operation. @@ -331,7 +696,15 @@ def capture_sources( docker_sources = tuple(source for source in sources if source.kind == "docker") if docker_sources and (docker_runtime is None or operation_id is None): raise CaptureError("Docker source capture requires the shared local Docker runtime") - with tempfile.TemporaryDirectory(prefix="capture-", dir=state_dir) as temporary: + with ExitStack() as temporary_stack: + if durable_artifacts is None: + temporary = Path(temporary_stack.enter_context( + tempfile.TemporaryDirectory(prefix="capture-", dir=state_dir))) + else: + durable_artifacts.prepare() + temporary = durable_artifacts.directory + if resource_monitor is not None: + resource_monitor.check() paths = [] exports = [] for source in sources: @@ -347,39 +720,82 @@ def capture_sources( original = Path(source.paths[0]) if not original.is_file(): raise CaptureError("SQLite source must be an existing database file") + covered_sqlite = (coverage_sqlite or {}).get(source.name) + if covered_sqlite is not None: + if (covered_sqlite.get("path") != str(original) + or covered_sqlite.get("source_id") != source.name + or not isinstance(covered_sqlite.get("identity"), dict)): + raise CaptureError("Reviewed SQLite mapping does not match the source") + try: + verify_sqlite_identity(original, covered_sqlite["identity"]) + except CoverageError: + raise CaptureError("Reviewed SQLite source identity changed") from None target = Path(temporary) / (source.name + ".sqlite") + if durable_artifacts is not None: + durable_artifacts.begin_export(target) def progress(status, remaining, total): if time.monotonic() >= deadline: raise CaptureError("SQLite capture timed out") if cancel is not None and cancel.is_set(): raise CaptureError("Capture cancelled") + if resource_monitor is not None: + resource_monitor.check() try: with sqlite3.connect(original.as_uri() + "?mode=ro", uri=True) as src: + if resource_monitor is not None: + page_size = src.execute("PRAGMA page_size").fetchone()[0] + page_count = src.execute("PRAGMA page_count").fetchone()[0] + resource_monitor.check(page_size * page_count) with sqlite3.connect(target) as dst: target.chmod(0o600) src.backup(dst, pages=256, progress=progress, sleep=0.05) + if resource_monitor is not None: + resource_monitor.check() if dst.execute("PRAGMA quick_check").fetchone() != ("ok",): raise CaptureError("SQLite capture validation failed") except sqlite3.Error as exc: raise CaptureError("SQLite capture failed") from exc + if durable_artifacts is not None: + durable_artifacts.finish_export(target) + if covered_sqlite is not None: + try: + verify_sqlite_identity(original, covered_sqlite["identity"]) + except CoverageError: + raise CaptureError("Reviewed SQLite source identity changed") from None paths.append(str(target)) - exports.append({ + export = { "source": source.name, "kind": source.kind, "format": "sqlite", "path": str(target), - }) + } + if covered_sqlite is not None: + digest = hashlib.sha256() + with target.open("rb") as exported: + for chunk in iter(lambda: exported.read(1024 * 1024), b""): + digest.update(chunk) + export.update({ + "coverage_root_id": covered_sqlite["root_id"], + "original_path": str(original), + "reviewed_identity": covered_sqlite["identity"], + "recipe_digest": covered_sqlite["recipe_digest"], + "baseline_digest": covered_sqlite["baseline_digest"], + "export_sha256": digest.hexdigest(), + }) + exports.append(export) elif source.kind in ("postgresql", "mysql", "mariadb"): binding = _database_binding(source, database_bindings) suffix = ".dump" if source.kind == "postgresql" else ".sql" target = Path(temporary) / f"{source.name}{suffix}" + if durable_artifacts is not None: + durable_artifacts.begin_export(target) if source.kind == "postgresql": database_options = _capture_postgresql( - source, binding, target, runner, deadline, cancel + source, binding, target, runner, deadline, cancel, resource_monitor ) else: mysql_metadata = _capture_mysql( - source, binding, target, runner, deadline, cancel + source, binding, target, runner, deadline, cancel, resource_monitor ) paths.append(str(target)) exports.append({ @@ -392,8 +808,19 @@ def progress(status, remaining, total): }) if source.kind == "postgresql": exports[-1]["database_options"] = database_options + reviewed = (coverage_postgresql or {}).get(source.name) + if reviewed is not None: + if (reviewed.get("source_id") != source.name + or reviewed.get("connection") != source.connection + or reviewed.get("database") != source.database): + raise CaptureError("Reviewed PostgreSQL mapping differs from export") + exports[-1]["coverage"] = dict(reviewed) + exports[-1]["source_database"] = source.database + exports[-1]["export_sha256"] = _file_sha256(target, resource_monitor) else: exports[-1].update(mysql_metadata) + if durable_artifacts is not None: + durable_artifacts.finish_export(target) elif source.kind != "docker": raise CaptureError("Source adapter is not implemented") @@ -409,9 +836,15 @@ def progress(status, remaining, total): paths.extend(docker_paths) exports.extend(docker_exports) if exports: + if validate_coverage is not None: + validate_coverage() manifest = Path(temporary) / "bbackup-capture.json" + if durable_artifacts is not None: + durable_artifacts.begin_export(manifest) with manifest.open("x", encoding="utf-8") as output: manifest.chmod(0o600) json.dump({"schema_version": 2, "exports": exports}, output) paths.append(str(manifest)) + if durable_artifacts is not None: + durable_artifacts.ready(manifest if exports else None) yield paths diff --git a/bbackup/cli_metadata.py b/bbackup/cli_metadata.py index b40a4cd..1f5fa21 100644 --- a/bbackup/cli_metadata.py +++ b/bbackup/cli_metadata.py @@ -1953,7 +1953,7 @@ class ProductionCommand(TypedDict): "config migrate": {"summary": "Write enrolled version-three files beside untouched version-two originals.", "fields": {}}, "service enroll": {"summary": "Explicitly enroll the exact version-three policy and bindings on this machine and user.", "fields": {}}, "jobs list": {"summary": "List configured capture jobs.", "fields": {}}, - "jobs run": {"summary": "Capture locally and copy explicit successful snapshots.", "fields": {"name": "string"}}, + "jobs run": {"summary": "Capture locally and copy explicit successful snapshots.", "fields": {"name": "string", "quiet_administrative_window_confirmed": "boolean"}}, "repositories list": {"summary": "List declared repositories without host credentials.", "fields": {}}, "repositories init": {"summary": "Initialize an encrypted repository.", "fields": {"name": "string", "source": "optional-string"}}, "repositories check": {"summary": "Read and verify repository data.", "fields": {"name": "string"}}, @@ -1961,9 +1961,50 @@ class ProductionCommand(TypedDict): "snapshots copy": {"summary": "Copy a successful snapshot into an independent repository.", "fields": {"source": "string", "destination": "string", "snapshot": "string"}}, "retention plan": {"summary": "Review a read-only local retention plan bound to current snapshot and replica evidence.", "fields": {"job": "string"}}, "retention apply": {"summary": "Apply only a reviewed local retention plan after explicit approval; never prunes repository data.", "fields": {"job": "string", "plan_token": "string", "approved": "boolean"}}, + "cloud-retention catalog-enroll": { + "summary": "Explicitly enroll a new central exact-version retention catalog; never replace a missing established catalog.", + "fields": {"repository": "string", "policy_file": "path", "grant_file": "path"}, + }, + "cloud-retention plan": { + "summary": "Plan noncurrent object-version deletion from independently accepted checkpoints and private authority.", + "fields": {"repository": "string", "policy_file": "path", "grant_file": "path", "records_file": "path", "quiescent_window_confirmed": "boolean"}, + }, + "cloud-retention apply": { + "summary": "Apply one reviewed exact-version plan under the shared repository lease; never replay uncertain deletes.", + "fields": {"repository": "string", "policy_file": "path", "grant_file": "path", "records_file": "path", "plan_id": "string", "plan_sha256": "string", "quiescent_window_confirmed": "boolean"}, + }, + "cloud-retention reconcile": { + "summary": "Reconcile a cloud-version deletion journal using read-only observations without replaying it.", + "fields": {"repository": "string", "policy_file": "path", "plan_file": "path"}, + }, + "cloud-maintenance catalog-enroll": { + "summary": "Explicitly initialize the central catalog under a current-repository maintenance grant.", + "fields": {"repository": "string", "policy_file": "path", "grant_file": "path"}, + }, + "cloud-maintenance records-enroll": { + "summary": "Explicitly register independently signed accepted recovery records without changing version-age evidence.", + "fields": {"repository": "string", "policy_file": "path", "grant_file": "path", "records_file": "path"}, + }, + "cloud-maintenance plan": { + "summary": "Plan exact accepted and unprotected snapshot removal using read-only Restic observations.", + "fields": {"repository": "string", "policy_file": "path", "grant_file": "path", "records_file": "path", "maintenance_bindings_file": "path", "quiescent_window_confirmed": "boolean"}, + }, + "cloud-maintenance apply": { + "summary": "Apply a reviewed snapshot-forget and no-repack prune plan under one repository lease.", + "fields": {"repository": "string", "policy_file": "path", "grant_file": "path", "records_file": "path", "maintenance_bindings_file": "path", "plan_id": "string", "plan_sha256": "string", "quiescent_window_confirmed": "boolean"}, + }, + "cloud-maintenance reconcile": { + "summary": "Observe an interrupted maintenance attempt without replaying forget or prune.", + "fields": {"repository": "string", "policy_file": "path", "grant_file": "path", "records_file": "path", "maintenance_bindings_file": "path", "plan_id": "string", "plan_sha256": "string", "quiescent_window_confirmed": "boolean"}, + }, "restore": {"summary": "Restore and verify files in a new isolated directory.", "fields": {"repository": "string", "snapshot": "string", "target": "path"}}, "runs list": {"summary": "Read sanitized operation events.", "fields": {"limit": "limit", "offset": "offset"}}, "service render": {"summary": "Render disabled systemd service and persistent timer units.", "fields": {"name": "string", "executable": "path"}}, + "service render-system": {"summary": "Render root system service and persistent timer units after validating root-controlled executable and configuration paths; never installs them.", "fields": {"name": "string", "executable": "path"}}, + "coverage discover": {"summary": "Inventory exact private Linux scope and record a bounded candidate for review.", "fields": {"name": "string", "quiet_administrative_window_confirmed": "boolean"}}, + "coverage review": {"summary": "Approve an exact complete Linux coverage candidate for the enrolled machine, user, policy and bindings.", "fields": {"name": "string", "candidate_digest": "string", "quiet_administrative_window_confirmed": "boolean"}}, + "coverage check": {"summary": "Check reviewed Linux coverage and structural drift without exposing private paths.", "fields": {"name": "string", "quiet_administrative_window_confirmed": "boolean"}}, + "coverage recovery-evidence": {"summary": "Assess private local recovery evidence against the exact current baseline and snapshot; never qualifies whole-machine recovery.", "fields": {"name": "string", "snapshot": "string", "evidence_file": "optional-string", "quiet_administrative_window_confirmed": "boolean"}}, "service install": {"summary": "Install reviewed user-level units; activate the exact timer only with --activate.", "fields": {"name": "string", "executable": "path", "activate": "boolean"}}, } @@ -1991,6 +2032,14 @@ def production_schema(command: str) -> dict: "recovery inspect": {"summary": "Inspect cloud versioning and lifecycle without a protection claim.", "fields": {"bucket": "string", "endpoint": "string", "prefix": "optional-string", "region": "optional-string"}}, "recovery checkpoint": {"summary": "Check a quiescent repository and seal an independent recovery kit.", "fields": {"repository": "string", "snapshot": "string", "bucket": "string", "endpoint": "string", "prefix": "optional-string", "region": "optional-string", "target": "path", "signer": "string", "recipient": "string", "quiescent_window_confirmed": "boolean"}}, "recovery open": {"summary": "Authenticate and decrypt a recovery kit into a new private directory.", "fields": {"kit": "path", "target": "path", "signer": "string"}}, + "recovery accept": { + "summary": "Independently reconstruct and check a historical repository, then sign central acceptance with a separate verifier key.", + "fields": { + "kit": "path", "target": "path", "source_signer": "string", "verifier": "string", + "repository": "string", "repository_id": "string", "snapshot": "string", + "bucket": "string", "endpoint": "string", "prefix": "optional-string", "region": "optional-string", + }, + }, "recovery restore": {"summary": "Restore signed exact historical object versions into a new local repository.", "fields": {"manifest": "path", "signature": "path", "signer": "string", "password_file": "path", "repository": "string", "repository_id": "string", "snapshot": "string", "bucket": "string", "endpoint": "string", "prefix": "optional-string", "region": "optional-string", "target": "path"}}, }) @@ -2025,7 +2074,16 @@ def production_schema(command: str) -> dict: PRODUCTION_COMMANDS["runs reconcile"] = { "summary": "Record an administrator-reviewed outcome for an uncertain operation without replaying it.", - "fields": {"repository": "string", "operation_id": "string", "outcome": "string", "repository_reviewed": "boolean"}, + "fields": {"repository": "string", "operation_id": "string", "outcome": "string", "repository_reviewed": "boolean", "quiescent_window_confirmed": "boolean"}, +} + +PRODUCTION_COMMANDS["runs retained-exports"] = { + "summary": "List bounded private remote capture artifact records, including captures stopped before dispatch; counts and digests are declared, not file-hash verification.", + "fields": {"repository": "string", "limit": "limit", "offset": "offset"}, +} +PRODUCTION_COMMANDS["runs discard-exports"] = { + "summary": "Explicitly discard the exact retained exports of a reviewed failed or abandoned remote capture, bound to its current private record digest.", + "fields": {"repository": "string", "operation_id": "string", "artifact_record_sha256": "string"}, } PRODUCTION_COMMANDS["databases restore"] = { diff --git a/bbackup/cloud.py b/bbackup/cloud.py index 5fcaf89..b2ee1a4 100644 --- a/bbackup/cloud.py +++ b/bbackup/cloud.py @@ -1,12 +1,13 @@ -"""Read-only S3-compatible version inventory and local checkpoint restore. +"""S3-compatible version inventory, checkpoint restore, and reviewed retention. -This module never writes to an object store and never creates a production -protection claim. Callers own quiescence coordination, signature trust, and any -manifest sealing workflow. +Retention deletion requires a private central policy and recurring grant, +verified central recovery records, a fresh immutable plan, and a coordinated +quiescent window. The module never creates a production protection claim. """ from __future__ import annotations +import fcntl import hashlib import ipaddress import json @@ -16,10 +17,13 @@ import sqlite3 import stat import tempfile -from collections.abc import Iterator, Mapping +import uuid +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta, timezone from threading import Event +from types import MappingProxyType from typing import Any from urllib.parse import urlsplit, urlunsplit @@ -86,6 +90,20 @@ class RestoreRecord: protection_expired: bool +@dataclass(frozen=True, slots=True) +class CloudMaintenanceSelection: + """Immutable accepted-snapshot protection and bounded S3 inventory summary.""" + + accepted_snapshot_ids: tuple[str, ...] + protected_snapshot_ids: tuple[str, ...] + protected_version_sha256: str + policy_sha256: str + grant_sha256: str + catalog_sha256: str + inventory_sha256: str + resources: Mapping[str, Any] + + def _iso(value: datetime) -> str: if value.tzinfo is None or value.utcoffset() is None: raise ValueError("timestamps must include a timezone") @@ -312,6 +330,8 @@ def __init__( region_name: str | None = None, prefix: str = "", page_size: int = DEFAULT_PAGE_SIZE, + native_lock_reader: Any | None = None, + credential_pair: tuple[str, str] | None = None, ) -> None: if ( not isinstance(bucket, str) @@ -341,6 +361,13 @@ def __init__( options["endpoint_url"] = configured_endpoint if region_name is not None: options["region_name"] = region_name + if credential_pair is not None: + if ( + not isinstance(credential_pair, tuple) or len(credential_pair) != 2 + or any(not isinstance(value, str) or not value for value in credential_pair) + ): + raise ValueError("credential_pair must be an explicit access-key pair") + options["aws_access_key_id"], options["aws_secret_access_key"] = credential_pair options["config"] = Config( connect_timeout=10, read_timeout=30, @@ -359,6 +386,17 @@ def __init__( configured_endpoint = _normalize_endpoint_url(client_endpoint) self.endpoint_url = configured_endpoint self.client = client + if credential_pair is not None and ( + not isinstance(credential_pair, tuple) or len(credential_pair) != 2 + or any(not isinstance(value, str) or not value for value in credential_pair) + ): + raise ValueError("credential_pair must be an explicit access-key pair") + if native_lock_reader is not None and credential_pair is None: + raise ValueError("native lock reader requires the same explicit S3 credential pair") + if native_lock_reader is not None and not native_lock_reader.matches_credentials(credential_pair): + raise ValueError("native lock reader credentials differ from S3 credentials") + self.native_lock_reader = native_lock_reader + self._credential_pair = credential_pair def inspect_lifecycle(self) -> LifecycleInspection: """Read bucket versioning and applicable lifecycle expiration rules.""" @@ -1041,10 +1079,3269 @@ def restore_checkpoint( ), ) + def plan_central_retention( + self, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + recovery_record_dirs: Sequence[str | os.PathLike[str]], + *, + quiescent_window_confirmed: bool, + repository_lease_id: str, + ) -> dict[str, Any]: + """Create a private immutable retention plan without deleting anything.""" + return _CentralRetention(self).plan( + policy_path, + grant_path, + recovery_record_dirs, + quiescent_window_confirmed=quiescent_window_confirmed, + repository_lease_id=repository_lease_id, + ) + + @contextmanager + def maintenance_session( + self, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + record_dirs: Sequence[str | os.PathLike[str]], + *, + quiescent_window_confirmed: bool, + repository_lease_id: str, + reconcile_only: bool = False, + ) -> Iterator[Any]: + """Hold the central catalog lock during one authorized logical maintenance.""" + with _CentralMaintenance( + self, + policy_path, + grant_path, + record_dirs, + quiescent_window_confirmed=quiescent_window_confirmed, + repository_lease_id=repository_lease_id, + reconcile_only=reconcile_only, + ) as session: + yield session + + def initialize_central_retention_catalog( + self, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + *, + expected_action: str = "delete_noncurrent_versions", + ) -> dict[str, Any]: + """Explicitly enroll a new private catalog before the first plan.""" + return _CentralRetention(self).initialize_catalog( + policy_path, grant_path, expected_action=expected_action + ) + + def enroll_central_maintenance_records( + self, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + recovery_record_dirs: Sequence[str | os.PathLike[str]], + ) -> dict[str, Any]: + """Explicitly register independently accepted records without observing versions.""" + return _CentralRetention(self).enroll_accepted_records( + policy_path, grant_path, recovery_record_dirs + ) + + def apply_central_retention( + self, + plan_id: str, + expected_plan_sha256: str, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + recovery_record_dirs: Sequence[str | os.PathLike[str]], + *, + quiescent_window_confirmed: bool, + ledger: Any, + cancel_event: Event | None = None, + mutation_client: Any | None = None, + expected_repository_lease_id: str | None = None, + ) -> dict[str, Any]: + """Apply one reviewed exact-version plan without replaying uncertainty.""" + return _CentralRetention(self).apply( + plan_id, + expected_plan_sha256, + policy_path, + grant_path, + recovery_record_dirs, + quiescent_window_confirmed=quiescent_window_confirmed, + ledger=ledger, + cancel_event=cancel_event, + mutation_client=mutation_client, + expected_repository_lease_id=expected_repository_lease_id, + ) + + def reconcile_central_retention( + self, + plan_path: str | os.PathLike[str], + *, + ledger: Any, + expected_repository_lease_id: str | None = None, + ) -> dict[str, Any]: + """Read exact planned version IDs and reconcile without deleting.""" + return _CentralRetention(self).reconcile( + plan_path, ledger=ledger, + expected_repository_lease_id=expected_repository_lease_id, + ) + + +_RETENTION_POLICY_FIELDS = { + "schema_version", + "policy_id", + "catalog_id", + "revision", + "endpoint_url", + "bucket", + "prefix", + "repository_id", + "trusted_verifier", + "retention_counts", + "grace_seconds", + "plan_max_age_seconds", + "max_apply_objects", + "max_apply_bytes", + "state_dir", + "known_good_manifest_digests", + "pending_manifest_digests", +} +_RETENTION_GRANT_FIELDS = { + "schema_version", + "policy_id", + "catalog_id", + "policy_revision", + "policy_sha256", + "endpoint_url", + "bucket", + "prefix", + "repository_id", + "action", + "expires_at", + "max_apply_objects", + "max_apply_bytes", + "stop_conditions", +} +_RETENTION_STOP_CONDITIONS = [ + "cancelled", + "input_drift", + "lock_unknown_or_changed", + "mutation_outcome_uncertain", + "policy_or_grant_expired", +] +_MIN_RETENTION_GRACE_SECONDS = 30 * 24 * 60 * 60 +_PRIVATE_JSON_LIMIT = 16 * 1024 * 1024 +_RETENTION_PLAN_FIELDS = { + "schema_version", + "operation_id", + "created_at", + "deadline", + "target", + "repository_lease_id", + "policy_id", + "policy_revision", + "policy_sha256", + "grant_sha256", + "grant_expires_at", + "apply_limits", + "inventory_sha256", + "version_count", + "catalog_sha256", + "accepted_manifest_digests", + "selected_manifest_digests", + "protected_sha256", + "protected_version_count", + "candidate_lock_sha256", + "held_count", + "bucket_evidence_sha256", + "object_lock_enabled", + "candidates", + "key_inventory", +} +_RETENTION_JOURNAL_FIELDS = { + "schema_version", + "operation_id", + "plan_sha256", + "status", + "updated_at", + "items", +} +_MAINTENANCE_MODE = "forget-and-prune-no-repack" +_MAINTENANCE_PHASES = ( + "check_before", + "forget", + "verify_snapshots", + "prune", + "check_after", + "protected_versions", +) +_MAINTENANCE_MUTATION_PHASES = frozenset({"forget", "prune"}) +_MAINTENANCE_PLAN_FIELDS = { + "schema_version", + "operation_kind", + "operation_id", + "created_at", + "deadline", + "target", + "repository_lease_id", + "policy_id", + "policy_revision", + "policy_sha256", + "grant_sha256", + "grant_expires_at", + "catalog_sha256", + "accepted_manifest_digests", + "selected_manifest_digests", + "accepted_snapshot_ids", + "protected_snapshot_ids", + "protected_sha256", + "protected_version_count", + "inventory_sha256", + "version_count", + "resource_inventory_sha256", + "resources", + "apply_limits", + "maintenance", + "config_repository_id", + "visible_snapshot_ids", + "snapshot_inventory_sha256", + "target_snapshot_ids", + "bindings_sha256", + "credential_identity_sha256", +} +_MAINTENANCE_JOURNAL_FIELDS = { + "schema_version", + "operation_kind", + "operation_id", + "plan_sha256", + "status", + "updated_at", + "phases", + "reconciliation", +} + + +def _maintenance_snapshot_ids(value: Any, *, label: str) -> list[str]: + if ( + not isinstance(value, list) + or any(not isinstance(item, str) or not _FULL_ID.fullmatch(item) for item in value) + or value != sorted(set(value)) + ): + raise CloudError(f"Central maintenance {label} must be sorted unique full snapshot IDs") + return list(value) + + +def _maintenance_snapshot_digest(repository_id: str, snapshot_ids: Sequence[str]) -> str: + return _retention_digest( + _retention_json_bytes( + { + "config_repository_id": repository_id, + "visible_snapshot_ids": list(snapshot_ids), + } + ) + ) + + +def _maintenance_restic_evidence( + value: Any, *, expected_repository_id: str +) -> dict[str, Any]: + if not isinstance(value, Mapping) or set(value) != { + "config_repository_id", + "visible_snapshot_ids", + "snapshot_inventory_sha256", + }: + raise CloudError("Central maintenance Restic evidence schema is invalid") + repository_id = value.get("config_repository_id") + if repository_id != expected_repository_id or not _FULL_ID.fullmatch(str(repository_id)): + raise CloudError("Central maintenance Restic repository identity differs from policy") + snapshot_ids = _maintenance_snapshot_ids( + value.get("visible_snapshot_ids"), label="visible snapshot inventory" + ) + digest = value.get("snapshot_inventory_sha256") + if ( + not isinstance(digest, str) + or not _FULL_ID.fullmatch(digest) + or digest != _maintenance_snapshot_digest(repository_id, snapshot_ids) + ): + raise CloudError("Central maintenance Restic snapshot inventory digest is invalid") + return { + "config_repository_id": repository_id, + "visible_snapshot_ids": snapshot_ids, + "snapshot_inventory_sha256": digest, + } + + +def _maintenance_credential_digest(value: Any) -> str: + if not isinstance(value, Mapping) or set(value) != {"key_id", "profile"}: + raise CloudError("Central maintenance credential identity schema is invalid") + key_id = value.get("key_id") + profile = value.get("profile") + if ( + not isinstance(key_id, str) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", key_id) + or not isinstance(profile, str) + or profile not in {"restricted", "management"} + ): + raise CloudError("Central maintenance credential identity is invalid") + return _retention_digest( + _retention_json_bytes({"key_id": key_id, "profile": profile}) + ) + + +def _retention_json_default(item: Any) -> str: + if isinstance(item, datetime): + return _iso(item) + raise TypeError("unsupported retention metadata") + + +def _retention_json_bytes(value: Mapping[str, Any]) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=_retention_json_default, + ) + + "\n" + ).encode("utf-8") + + +def _retention_digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _retention_service_fields(response: Mapping[str, Any]) -> dict[str, Any]: + return {key: value for key, value in response.items() if key != "ResponseMetadata"} + + +def _no_symlink_path(path: Path, *, include_leaf: bool = True) -> None: + if not path.is_absolute() or ".." in path.parts: + raise CloudError("Retention paths must be absolute and traversal-free") + parts = path.parts if include_leaf else path.parts[:-1] + current = Path(path.anchor) + for part in parts[1:]: + current = current / part + try: + info = current.lstat() + except FileNotFoundError: + continue + if stat.S_ISLNK(info.st_mode): + raise CloudError("Retention paths cannot contain symlinks") + + +def _ensure_private_directory(path: Path, *, create: bool = False) -> Path: + path = path.absolute() + _no_symlink_path(path) + try: + info = path.lstat() + except FileNotFoundError: + if not create: + raise CloudError("Private retention state directory is missing") from None + path.mkdir(mode=0o700, parents=True, exist_ok=True) + _no_symlink_path(path) + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o077: + raise CloudError("Retention state directory must be private") + return path + + +def _read_private_json(path: Path, *, limit: int = _PRIVATE_JSON_LIMIT) -> tuple[dict[str, Any], bytes]: + path = path.absolute() + _no_symlink_path(path) + descriptor = -1 + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW) + info = os.fstat(descriptor) + if ( + not stat.S_ISREG(info.st_mode) + or stat.S_IMODE(info.st_mode) & 0o077 + or info.st_size > limit + ): + raise CloudError("Retention metadata must be a bounded private file") + chunks = [] + size = 0 + while True: + chunk = os.read(descriptor, min(1024 * 1024, limit + 1 - size)) + if not chunk: + break + chunks.append(chunk) + size += len(chunk) + if size > limit: + raise CloudError("Retention metadata exceeds its size limit") + raw = b"".join(chunks) + def unique_pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate key") + result[key] = value + return result + value = json.loads(raw.decode("utf-8"), object_pairs_hook=unique_pairs) + except CloudError: + raise + except (OSError, UnicodeDecodeError, ValueError, RecursionError): + raise CloudError("Retention metadata could not be read or parsed") from None + finally: + if descriptor >= 0: + os.close(descriptor) + if not isinstance(value, dict): + raise CloudError("Retention metadata must be a JSON object") + return value, raw + + +def _write_private_json(path: Path, value: Mapping[str, Any], *, replace: bool) -> bytes: + raw = _retention_json_bytes(value) + _ensure_private_directory(path.parent) + _no_symlink_path(path) + descriptor = -1 + temporary = None + try: + if replace: + descriptor, name = tempfile.mkstemp(prefix=".retention-", dir=path.parent) + temporary = Path(name) + else: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.fchmod(descriptor, 0o600) + view = memoryview(raw) + while view: + written = os.write(descriptor, view) + view = view[written:] + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + if replace: + os.replace(temporary, path) + temporary = None + parent_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(parent_fd) + finally: + os.close(parent_fd) + except CloudError: + raise + except OSError: + raise CloudError("Could not durably write private retention state") from None + finally: + if descriptor >= 0: + os.close(descriptor) + if temporary is not None: + try: + temporary.unlink() + except OSError: + pass + return raw + + +def _write_immutable_plan(path: Path, value: Mapping[str, Any]) -> bytes: + raw = _retention_json_bytes(value) + _ensure_private_directory(path.parent) + _no_symlink_path(path) + descriptor = -1 + created = False + complete = False + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o400) + created = True + os.fchmod(descriptor, 0o400) + view = memoryview(raw) + while view: + written = os.write(descriptor, view) + view = view[written:] + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + parent_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(parent_fd) + finally: + os.close(parent_fd) + complete = True + except OSError: + raise CloudError("Could not durably create immutable retention plan") from None + finally: + if descriptor >= 0: + os.close(descriptor) + if created and not complete and os.path.lexists(path): + try: + path.unlink() + except OSError: + pass + return raw + + +def _validate_retention_policy( + policy: Mapping[str, Any], adapter: S3CloudAdapter +) -> tuple[dict[str, Any], Path]: + version = policy.get("schema_version") + if type(version) is not int or version not in (1, 2, 3): + raise CloudError("Unsupported central retention policy version") + expected_fields = set(_RETENTION_POLICY_FIELDS) + if version == 2 or (version == 3 and "lock_metadata" in policy): + expected_fields.add("lock_metadata") + if version == 3: + expected_fields.add("maintenance") + if set(policy) != expected_fields: + raise CloudError("Central retention policy schema is invalid") + maintenance = None + if version == 3: + maintenance_value = policy.get("maintenance") + if ( + not isinstance(maintenance_value, dict) + or set(maintenance_value) != {"mode", "resource_budget", "timeout_seconds"} + or maintenance_value.get("mode") != "forget-and-prune-no-repack" + or type(maintenance_value.get("timeout_seconds")) is not int + or not 1 <= maintenance_value["timeout_seconds"] <= 86400 + ): + raise CloudError("Central logical-maintenance policy is invalid") + try: + from .models import ResourceBudget + + budget = ResourceBudget.parse(maintenance_value["resource_budget"]) + except Exception: + raise CloudError("Central logical-maintenance resource budget is invalid") from None + maintenance = { + "mode": "forget-and-prune-no-repack", + "resource_budget": { + "schema_version": 1, + "staging_max_bytes": budget.staging_max_bytes, + "minimum_free_bytes": budget.minimum_free_bytes, + "cache_max_bytes": budget.cache_max_bytes, + }, + "timeout_seconds": maintenance_value["timeout_seconds"], + } + from .b2 import B2VersionLockReader + + reader = adapter.native_lock_reader + if version == 1 and reader is not None: + raise CloudError("Schema-1 retention cannot use a native lock reader") + if version == 3 and "lock_metadata" not in policy and reader is not None: + raise CloudError("Schema-3 retention without lock metadata cannot use a native lock reader") + if version == 2 or (version == 3 and "lock_metadata" in policy): + metadata = policy["lock_metadata"] + if ( + not isinstance(metadata, dict) or set(metadata) != {"provider", "bucket_id"} + or metadata.get("provider") != "b2-native" + or not isinstance(metadata.get("bucket_id"), str) + or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", metadata["bucket_id"]) + or not isinstance(reader, B2VersionLockReader) + or reader.bucket != adapter.bucket or reader.bucket_id != metadata["bucket_id"] + or reader.prefix != adapter.prefix or reader.endpoint_url != adapter.endpoint_url + ): + raise CloudError("Native lock reader differs from schema-2 retention target") + if not isinstance(policy.get("policy_id"), str) or not _SOURCE_NAME.fullmatch(policy["policy_id"]): + raise CloudError("Central retention policy identity is invalid") + if not isinstance(policy.get("catalog_id"), str) or not _FULL_ID.fullmatch(policy["catalog_id"]): + raise CloudError("Central retention catalog identity is invalid") + if type(policy.get("revision")) is not int or policy["revision"] < 1: + raise CloudError("Central retention policy revision is invalid") + try: + endpoint = _normalize_endpoint_url(policy.get("endpoint_url")) + except ValueError: + raise CloudError("Central retention endpoint is invalid") from None + if endpoint != adapter.endpoint_url or policy.get("bucket") != adapter.bucket: + raise CloudError("Central retention target differs from the S3 adapter") + if ( + not isinstance(policy.get("prefix"), str) + or not policy["prefix"] + or policy["prefix"] != adapter.prefix + ): + raise CloudError("Central retention prefix differs from the S3 adapter") + _repository_identity(policy.get("repository_id")) + try: + from .recovery import _fingerprint + + verifier = _fingerprint(policy.get("trusted_verifier")) + except Exception: + raise CloudError("Central retention verifier fingerprint is invalid") from None + counts = policy.get("retention_counts") + if not isinstance(counts, dict) or set(counts) != {"daily", "weekly", "monthly"}: + raise CloudError("Central retention checkpoint counts are invalid") + if any(type(counts[name]) is not int or counts[name] < 0 for name in counts): + raise CloudError("Central retention checkpoint counts are invalid") + for name, minimum in ( + ("grace_seconds", _MIN_RETENTION_GRACE_SECONDS), + ("plan_max_age_seconds", 1), + ("max_apply_objects", 1), + ("max_apply_bytes", 1), + ): + if type(policy.get(name)) is not int or policy[name] < minimum: + raise CloudError("Central retention limits are invalid") + digests = {} + for name in ("known_good_manifest_digests", "pending_manifest_digests"): + values = policy.get(name) + if ( + not isinstance(values, list) + or any(not isinstance(value, str) or not _FULL_ID.fullmatch(value) for value in values) + or len(set(values)) != len(values) + ): + raise CloudError("Central retention manifest pins are invalid") + digests[name] = list(values) + if not digests["known_good_manifest_digests"]: + raise CloudError("Central retention requires an explicit known-good recovery checkpoint pin") + state_value = policy.get("state_dir") + if not isinstance(state_value, str) or not Path(state_value).is_absolute(): + raise CloudError("Central retention state directory must be absolute") + state_dir = _ensure_private_directory(Path(state_value), create=True) + clean = dict(policy) + clean["endpoint_url"] = endpoint + clean["trusted_verifier"] = verifier + clean.update(digests) + if maintenance is not None: + clean["maintenance"] = maintenance + return clean, state_dir + + +def _validate_retention_grant( + grant: Mapping[str, Any], + policy: Mapping[str, Any], + *, + policy_digest: str, + now: datetime, + expected_action: str = "delete_noncurrent_versions", + allow_expired: bool = False, +) -> tuple[dict[str, Any], datetime]: + if set(grant) != _RETENTION_GRANT_FIELDS: + raise CloudError("Recurring retention grant schema is invalid") + version = grant.get("schema_version") + if type(version) is not int or version not in (1, 2): + raise CloudError("Unsupported recurring retention grant version") + if expected_action not in {"delete_noncurrent_versions", "maintain_current_repository"}: + raise CloudError("Unsupported central cloud operation") + if version == 1 and expected_action != "delete_noncurrent_versions": + raise CloudError("Schema-1 recurring grants authorize deletion only") + if grant.get("action") not in {"delete_noncurrent_versions", "maintain_current_repository"}: + raise CloudError("Recurring retention grant action is invalid") + expected = { + "policy_id": policy["policy_id"], + "catalog_id": policy["catalog_id"], + "policy_revision": policy["revision"], + "policy_sha256": policy_digest, + "endpoint_url": policy["endpoint_url"], + "bucket": policy["bucket"], + "prefix": policy["prefix"], + "repository_id": policy["repository_id"], + "action": expected_action, + } + if any(grant.get(key) != value for key, value in expected.items()): + raise CloudError("Recurring retention grant does not match this policy target") + try: + expires = _parse_time(grant.get("expires_at")) + except CloudError: + raise CloudError("Recurring retention grant expiry is invalid") from None + if expires <= now and not allow_expired: + raise CloudError("Recurring retention grant has expired") + for name in ("max_apply_objects", "max_apply_bytes"): + value = grant.get(name) + if type(value) is not int or value < 1 or value > policy[name]: + raise CloudError("Recurring retention grant limits are invalid") + if grant.get("stop_conditions") != _RETENTION_STOP_CONDITIONS: + raise CloudError("Recurring retention grant stop conditions are invalid") + return dict(grant), expires + + +def _retention_version_token(key: str, version_id: str) -> str: + return _retention_digest(_retention_json_bytes({"key": key, "version_id": version_id})) + + +def _retention_target(policy: Mapping[str, Any]) -> dict[str, str]: + return { + "endpoint_url": policy["endpoint_url"], + "bucket": policy["bucket"], + "prefix": policy["prefix"], + "repository_id": policy["repository_id"], + } + + +def _validate_catalog(catalog: Mapping[str, Any], policy: Mapping[str, Any]) -> dict[str, Any]: + if set(catalog) != { + "schema_version", + "catalog_id", + "target", + "accepted_records", + "first_observed_noncurrent", + } or type(catalog.get("schema_version")) is not int or catalog["schema_version"] != 1: + raise CloudError("Central retention catalog schema is invalid") + if catalog.get("target") != _retention_target(policy): + raise CloudError("Central retention catalog belongs to a different target") + if catalog.get("catalog_id") != policy["catalog_id"]: + raise CloudError("Central retention catalog identity differs from authority") + accepted = catalog.get("accepted_records") + first_seen = catalog.get("first_observed_noncurrent") + if not isinstance(accepted, dict) or not isinstance(first_seen, dict): + raise CloudError("Central retention catalog is invalid") + for digest, entry in accepted.items(): + if ( + not isinstance(digest, str) + or not _FULL_ID.fullmatch(digest) + or not isinstance(entry, dict) + or set(entry) != {"snapshot_id", "verified_at", "source_repository", "captured_at", "expires_at", "source_signer"} + ): + raise CloudError("Central retention catalog is invalid") + if not isinstance(entry["snapshot_id"], str) or not _FULL_ID.fullmatch(entry["snapshot_id"]): + raise CloudError("Central retention catalog is invalid") + if not isinstance(entry["source_repository"], str) or not _SOURCE_NAME.fullmatch(entry["source_repository"]): + raise CloudError("Central retention catalog is invalid") + if not isinstance(entry["source_signer"], str) or not re.fullmatch(r"[0-9A-F]{40}|[0-9A-F]{64}", entry["source_signer"]): + raise CloudError("Central retention catalog is invalid") + for time_name in ("verified_at", "captured_at"): + _parse_time(entry[time_name]) + if entry["expires_at"] is not None: + _parse_time(entry["expires_at"]) + for token, timestamp in first_seen.items(): + if not isinstance(token, str) or not _FULL_ID.fullmatch(token): + raise CloudError("Central retention catalog is invalid") + if timestamp is not None: + _parse_time(timestamp) + return dict(catalog) + + +def _select_retention_records( + records: Sequence[Mapping[str, Any]], counts: Mapping[str, int] +) -> set[str]: + selected: set[str] = set() + periods = ( + ("daily", lambda value: value.strftime("%Y-%m-%d")), + ("weekly", lambda value: f"{value.isocalendar().year:04d}-W{value.isocalendar().week:02d}"), + ("monthly", lambda value: value.strftime("%Y-%m")), + ) + for name, bucket_for in periods: + buckets: dict[str, list[Mapping[str, Any]]] = {} + for record in records: + verified = _parse_time(record["verified_at"]) + buckets.setdefault(bucket_for(verified), []).append(record) + ordered = sorted( + ( + min(items, key=lambda item: (_parse_time(item["verified_at"]), item["manifest_sha256"])) + for items in buckets.values() + ), + key=lambda item: (_parse_time(item["verified_at"]), item["manifest_sha256"]), + ) + count = counts[name] + if count: + selected.update(item["manifest_sha256"] for item in ordered[-count:]) + return selected + + +@contextmanager +def _retention_catalog_lock(state_dir: Path): + path = state_dir / "catalog.lock" + descriptor = -1 + try: + descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + os.fchmod(descriptor, 0o600) + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + except OSError: + raise CloudError("Could not acquire the private retention catalog lock") from None + finally: + if descriptor >= 0: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + + +class _CentralRetention: + """Private reviewed retention workflow owned by the S3 cloud adapter.""" + + def __init__(self, adapter: S3CloudAdapter) -> None: + self.adapter = adapter + + def _authority( + self, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + *, + expected_action: str = "delete_noncurrent_versions", + allow_expired: bool = False, + ) -> tuple[dict[str, Any], dict[str, Any], Path, datetime, str, str, Path, Path]: + policy_file = Path(policy_path).absolute() + grant_file = Path(grant_path).absolute() + policy, policy_raw = _read_private_json(policy_file) + clean_policy, state_dir = _validate_retention_policy(policy, self.adapter) + grant, grant_raw = _read_private_json(grant_file) + now = datetime.now(timezone.utc) + clean_grant, grant_expiry = _validate_retention_grant( + grant, + clean_policy, + policy_digest=_retention_digest(policy_raw), + now=now, + expected_action=expected_action, + allow_expired=allow_expired, + ) + return ( + clean_policy, + clean_grant, + state_dir, + now, + _retention_digest(policy_raw), + _retention_digest(grant_raw), + policy_file, + grant_file, + ) + + def _assert_live_authority( + self, + plan: Mapping[str, Any], + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + ) -> None: + if datetime.now(timezone.utc) >= _parse_time(plan["deadline"]): + raise CloudError("Central retention plan has expired") + latest = self._authority(policy_path, grant_path) + if latest[4] != plan["policy_sha256"] or latest[5] != plan["grant_sha256"]: + raise CloudError("Central retention authority changed") + if datetime.now(timezone.utc) >= _parse_time(plan["deadline"]): + raise CloudError("Central retention plan has expired") + + def _pin_records( + self, + recovery_record_dirs: Sequence[str | os.PathLike[str]], + policy: Mapping[str, Any], + temporary: Path, + now: datetime, + ) -> list[dict[str, Any]]: + from . import recovery + + if not isinstance(recovery_record_dirs, Sequence) or isinstance( + recovery_record_dirs, (str, bytes) + ) or not recovery_record_dirs: + raise CloudError("At least one accepted recovery record is required") + records = [] + seen_digests = set() + for index, source_value in enumerate(recovery_record_dirs): + try: + source_dir = Path(source_value).absolute() + except (TypeError, ValueError): + raise CloudError("Recovery record directory is invalid") from None + _ensure_private_directory(source_dir) + pinned_dir = temporary / f"record-{index:06d}" + pinned_dir.mkdir(mode=0o700) + try: + for filename, limit in ( + ("acceptance.json", 16384), + ("acceptance.sig", 1024**2), + ("checkpoint.sig", 1024**2), + ): + recovery._copy( + source_dir / filename, + pinned_dir / filename, + limit=limit, + ) + candidate, _candidate_raw = _read_private_json( + pinned_dir / "acceptance.json", limit=16384 + ) + snapshot = candidate.get("snapshot_id") + if ( + candidate.get("repository_id") != policy["repository_id"] + or not isinstance(snapshot, str) + or not _FULL_ID.fullmatch(snapshot) + ): + raise CloudError("Recovery acceptance identity is invalid") + verified = recovery.verify_recovery_acceptance( + pinned_dir, + trusted_verifier=policy["trusted_verifier"], + repository_id=policy["repository_id"], + snapshot=snapshot, + ) + if verified != candidate: + raise CloudError("Recovery acceptance changed while being verified") + manifest_path = pinned_dir / "checkpoint.jsonl" + manifest_digest = self.adapter._pin_manifest( + source_dir / "checkpoint.jsonl", manifest_path + ) + if manifest_digest != verified.get("manifest_sha256"): + raise CloudError("Recovery manifest differs from its central acceptance") + signature_digest = recovery.verify_checkpoint( + manifest_path, + pinned_dir / "checkpoint.sig", + verified["source_signer"], + ) + if signature_digest != manifest_digest: + raise CloudError("Recovery source signature does not match its manifest") + index_path = temporary / f"manifest-index-{index:06d}.sqlite3" + connection, header, object_count, _marker_count = self.adapter._validate_manifest( + manifest_path, + index_path, + expected_source_repository=verified["source_repository"], + expected_repository_id=policy["repository_id"], + expected_source_snapshot_id=snapshot, + ) + try: + connection.commit() + manifest_bytes = connection.execute( + "SELECT COALESCE(SUM(size), 0) FROM objects WHERE kind = 'object'" + ).fetchone()[0] + finally: + connection.close() + verified_time = _parse_time(verified["verified_at"]) + if verified_time > now: + raise CloudError("Recovery acceptance is future-dated") + if ( + _parse_time(header["captured_at"]) + != _parse_time(verified["captured_at"]) + or header["expires_at"] != verified["expires_at"] + or object_count != verified["files_restored"] + or manifest_bytes != verified["bytes_restored"] + ): + raise CloudError("Recovery manifest does not match its acceptance record") + if manifest_digest in seen_digests: + raise CloudError("Duplicate recovery manifest was supplied") + seen_digests.add(manifest_digest) + records.append( + { + "manifest_sha256": manifest_digest, + "snapshot_id": snapshot, + "verified_at": _iso(verified_time), + "source_repository": verified["source_repository"], + "captured_at": _iso(_parse_time(verified["captured_at"])), + "expires_at": ( + _iso(_parse_time(verified["expires_at"])) + if verified["expires_at"] is not None + else None + ), + "source_signer": verified["source_signer"], + "index_path": str(index_path), + } + ) + except CloudError: + raise + except Exception: + raise CloudError("Central recovery record could not be verified") from None + return records + + @staticmethod + def _catalog_path(state_dir: Path) -> Path: + return state_dir / "retention-catalog.json" + + def _load_catalog( + self, state_dir: Path, policy: Mapping[str, Any] + ) -> tuple[dict[str, Any], bytes]: + path = self._catalog_path(state_dir) + if not os.path.lexists(path): + raise CloudError("Central retention catalog is missing; explicit enrollment is required") + catalog, raw = _read_private_json(path) + return _validate_catalog(catalog, policy), raw + + def initialize_catalog( + self, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + *, + expected_action: str = "delete_noncurrent_versions", + ) -> dict[str, Any]: + policy, _grant, state_dir, _now, _policy_sha, _grant_sha = self._authority( + policy_path, grant_path, expected_action=expected_action + )[:6] + with _retention_catalog_lock(state_dir): + policy = self._authority( + policy_path, grant_path, expected_action=expected_action + )[0] + path = self._catalog_path(state_dir) + if os.path.lexists(path): + raise CloudError("Central retention catalog is already enrolled") + for name in ("retention-plans", "retention-operations"): + directory = state_dir / name + if os.path.lexists(directory): + _ensure_private_directory(directory) + if any(directory.iterdir()): + raise CloudError("Central retention history exists; lost catalog cannot be re-enrolled") + catalog = { + "schema_version": 1, + "catalog_id": policy["catalog_id"], + "target": _retention_target(policy), + "accepted_records": {}, + "first_observed_noncurrent": {}, + } + raw = _write_private_json(path, catalog, replace=False) + return {"status": "enrolled", "catalog_id": policy["catalog_id"], "catalog_sha256": _retention_digest(raw)} + + def enroll_accepted_records( + self, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + recovery_record_dirs: Sequence[str | os.PathLike[str]], + ) -> dict[str, Any]: + """Reuse catalog acceptance, leaving historical first-seen evidence untouched.""" + policy, _grant, state_dir, _now, _policy_sha, _grant_sha = self._authority( + policy_path, grant_path, expected_action="maintain_current_repository" + )[:6] + with _retention_catalog_lock(state_dir): + policy, _grant, state_dir, now = self._authority( + policy_path, grant_path, expected_action="maintain_current_repository" + )[:4] + catalog, old_raw = self._load_catalog(state_dir, policy) + with tempfile.TemporaryDirectory(prefix="maintenance-records-", dir=state_dir) as name: + temporary = Path(name) + os.chmod(temporary, 0o700) + records = self._pin_records(recovery_record_dirs, policy, temporary, now) + self._merge_catalog_records(catalog, records, policy) + if _retention_json_bytes(catalog) == old_raw: + return {"status": "unchanged", "accepted_record_count": len(catalog["accepted_records"]), + "catalog_sha256": _retention_digest(old_raw)} + raw = _write_private_json(self._catalog_path(state_dir), catalog, replace=True) + return {"status": "enrolled", "accepted_record_count": len(catalog["accepted_records"]), + "catalog_sha256": _retention_digest(raw)} + + def _merge_catalog_records( + self, + catalog: dict[str, Any], + records: Sequence[Mapping[str, Any]], + policy: Mapping[str, Any], + ) -> None: + current = { + record["manifest_sha256"]: { + "snapshot_id": record["snapshot_id"], + "verified_at": record["verified_at"], + "source_repository": record["source_repository"], + "captured_at": record["captured_at"], + "expires_at": record["expires_at"], + "source_signer": record["source_signer"], + } + for record in records + } + previous = catalog["accepted_records"] + if set(previous) - set(current): + raise CloudError("A previously accepted recovery record is missing from the catalog input") + snapshot_digests = { + entry["snapshot_id"]: digest for digest, entry in previous.items() + } + for digest, entry in current.items(): + old = previous.get(digest) + if old is not None and old != entry: + raise CloudError("A recovery acceptance was replaced or re-accepted") + old_digest = snapshot_digests.get(entry["snapshot_id"]) + if old_digest is not None and old_digest != digest: + raise CloudError("A previously accepted snapshot was re-accepted with a new manifest") + merged = dict(previous) + merged.update(current) + pinned = set(policy["known_good_manifest_digests"]) | set( + policy["pending_manifest_digests"] + ) + if pinned - set(merged): + raise CloudError("A policy-pinned recovery manifest is missing") + catalog["accepted_records"] = merged + + def _observe_noncurrent( + self, + catalog: dict[str, Any], + inventory: sqlite3.Connection, + *, + now: datetime, + ) -> None: + first_seen = catalog["first_observed_noncurrent"] + for key, version_id, kind, is_latest in inventory.execute( + "SELECT key, version_id, kind, is_latest FROM versions ORDER BY key, version_id" + ): + if kind != "object": + continue + token = _retention_version_token(key, version_id) + if is_latest: + first_seen[token] = None + continue + value = first_seen.get(token) + if value is None: + first_seen[token] = _iso(now) + else: + timestamp = _parse_time(value) + if timestamp > now: + raise CloudError("Noncurrent-version grace catalog is future-dated") + + def _inventory(self, database_path: Path) -> tuple[sqlite3.Connection, str, int]: + connection = sqlite3.connect(database_path) + connection.execute( + """CREATE TABLE versions ( + key TEXT NOT NULL, + version_id TEXT NOT NULL, + kind TEXT NOT NULL, + size INTEGER, + is_latest INTEGER NOT NULL, + PRIMARY KEY(key, version_id) + )""" + ) + key_marker = None + version_marker = None + page_markers = set() + try: + while True: + marker_pair = (key_marker or "", version_marker or "") + if marker_pair in page_markers: + raise CloudError("Object-version pagination repeated a marker") + page_markers.add(marker_pair) + request = { + "Bucket": self.adapter.bucket, + "Prefix": self.adapter.prefix, + "MaxKeys": self.adapter.page_size, + } + if key_marker is not None: + request["KeyMarker"] = key_marker + if version_marker is not None: + request["VersionIdMarker"] = version_marker + try: + response = self.adapter.client.list_object_versions(**request) + except Exception: + raise CloudError("Could not read central retention version inventory") from None + if not isinstance(response, dict): + raise CloudError("Central retention version inventory is malformed") + for collection, kind in (("Versions", "object"), ("DeleteMarkers", "delete_marker")): + values = response.get(collection, []) + if not isinstance(values, list): + raise CloudError("Central retention version inventory is malformed") + for item in values: + if not isinstance(item, dict) or type(item.get("IsLatest")) is not bool: + raise CloudError("Central retention version inventory is malformed") + key = item.get("Key") + version_id = item.get("VersionId") + _relative_key(key, self.adapter.prefix) + if not isinstance(version_id, str) or not version_id: + raise CloudError("Central retention version ID is missing") + size = item.get("Size") if kind == "object" else None + if kind == "object" and (type(size) is not int or size < 0): + raise CloudError("Central retention version size is invalid") + try: + connection.execute( + "INSERT INTO versions VALUES (?, ?, ?, ?, ?)", + (key, version_id, kind, size, int(item["IsLatest"])), + ) + except sqlite3.IntegrityError: + raise CloudError("Central retention inventory contains duplicate versions") from None + truncated = response.get("IsTruncated", False) + if type(truncated) is not bool: + raise CloudError("Central retention inventory pagination state is invalid") + if not truncated: + break + next_key = response.get("NextKeyMarker") + next_version = response.get("NextVersionIdMarker") + if not isinstance(next_key, str) or not next_key: + raise CloudError("Truncated central inventory lacks a key marker") + if next_version is not None and not isinstance(next_version, str): + raise CloudError("Central inventory version marker is invalid") + next_pair = (next_key, next_version or "") + if next_pair == marker_pair: + raise CloudError("Central inventory pagination did not progress") + key_marker, version_marker = next_key, next_version + malformed_latest = connection.execute( + """SELECT key FROM versions GROUP BY key + HAVING SUM(is_latest) != 1 LIMIT 1""" + ).fetchone() + if malformed_latest is not None: + raise CloudError("Central retention inventory has an invalid latest-version set") + connection.commit() + digest = hashlib.sha256() + count = 0 + for row in connection.execute( + "SELECT key, version_id, kind, size, is_latest FROM versions ORDER BY key, version_id" + ): + digest.update(_retention_json_bytes({ + "key": row[0], "version_id": row[1], "kind": row[2], + "size": row[3], "is_latest": bool(row[4]), + })) + count += 1 + return connection, digest.hexdigest(), count + except BaseException: + connection.close() + raise + + def _bucket_evidence(self) -> dict[str, Any]: + client = self.adapter.client + try: + versioning = client.get_bucket_versioning(Bucket=self.adapter.bucket) + except Exception: + raise CloudError("Could not inspect central bucket versioning") from None + if not isinstance(versioning, dict) or versioning.get("Status") != "Enabled": + raise CloudError("Central retention requires enabled bucket versioning") + if set(versioning) - {"Status", "MFADelete", "ResponseMetadata"}: + raise CloudError("Central bucket versioning evidence contains unknown fields") + if versioning.get("MFADelete") not in (None, "Disabled", "Enabled"): + raise CloudError("Central bucket MFA-delete evidence is unknown") + if versioning.get("MFADelete") == "Enabled": + raise CloudError("Central retention does not support MFA Delete buckets") + try: + lifecycle = client.get_bucket_lifecycle_configuration(Bucket=self.adapter.bucket) + except Exception as exc: + if _error_code(exc) == "NoSuchLifecycleConfiguration": + lifecycle = {"Rules": [], "status": "NoSuchLifecycleConfiguration"} + else: + raise CloudError("Could not inspect central bucket lifecycle") from None + if not isinstance(lifecycle, dict) or not isinstance(lifecycle.get("Rules", []), list): + raise CloudError("Central bucket lifecycle evidence is malformed") + versioning = _retention_service_fields(versioning) + lifecycle = _retention_service_fields(lifecycle) + if set(lifecycle) - {"Rules", "TransitionDefaultMinimumObjectSize", "status"}: + raise CloudError("Central bucket lifecycle evidence contains unknown fields") + rules = lifecycle.get("Rules", []) + for rule in rules: + if ( + not isinstance(rule, dict) + or rule.get("Status") not in ("Enabled", "Disabled") + or set(rule) + - { + "ID", + "Prefix", + "Filter", + "Status", + "Expiration", + "Transitions", + "NoncurrentVersionTransitions", + "NoncurrentVersionExpiration", + "AbortIncompleteMultipartUpload", + "ExpiredObjectDeleteMarker", + } + ): + raise CloudError("Central bucket lifecycle evidence is malformed") + if rule["Status"] == "Enabled" and any( + name in rule for name in ( + "Expiration", "NoncurrentVersionExpiration", "ExpiredObjectDeleteMarker" + ) + ): + raise CloudError("Enabled bucket lifecycle expiration blocks central retention") + try: + object_lock = client.get_object_lock_configuration(Bucket=self.adapter.bucket) + if not isinstance(object_lock, dict): + raise CloudError("Central object-lock evidence is malformed") + object_lock = _retention_service_fields(object_lock) + if set(object_lock) != {"ObjectLockConfiguration"}: + raise CloudError("Central object-lock evidence contains unknown fields") + config = object_lock.get("ObjectLockConfiguration") + if ( + not isinstance(config, dict) + or config.get("ObjectLockEnabled") != "Enabled" + or set(config) - {"ObjectLockEnabled", "Rule"} + ): + raise CloudError("Central object-lock configuration is unknown or disabled") + rule = config.get("Rule") + if rule is not None: + if not isinstance(rule, dict) or set(rule) != {"DefaultRetention"}: + raise CloudError("Central default object-lock evidence is malformed") + default_retention = rule["DefaultRetention"] + if ( + not isinstance(default_retention, dict) + or default_retention.get("Mode") not in {"GOVERNANCE", "COMPLIANCE"} + or set(default_retention) not in ({"Mode", "Days"}, {"Mode", "Years"}) + or type(default_retention.get("Days", default_retention.get("Years"))) is not int + or default_retention.get("Days", default_retention.get("Years")) < 1 + ): + raise CloudError("Central default object-lock evidence is malformed") + lock_enabled = True + except CloudError: + raise + except Exception as exc: + if _error_code(exc) in {"ObjectLockConfigurationNotFoundError", "NoSuchObjectLockConfiguration"}: + object_lock = {"status": _error_code(exc)} + lock_enabled = False + else: + raise CloudError("Could not inspect central object-lock configuration") from None + return { + "versioning": versioning, + "lifecycle": lifecycle, + "object_lock": object_lock, + "object_lock_enabled": lock_enabled, + "digests": { + "versioning_sha256": _retention_digest(_retention_json_bytes(versioning)), + "lifecycle_sha256": _retention_digest(_retention_json_bytes(lifecycle)), + "object_lock_sha256": _retention_digest(_retention_json_bytes(object_lock)), + }, + } + + def _version_lock(self, key: str, version_id: str, *, lock_enabled: bool, now: datetime) -> tuple[dict[str, Any], bool]: + if self.adapter.native_lock_reader is not None: + try: + return self.adapter.native_lock_reader.read(key, version_id, now=now) + except Exception: + raise CloudError("B2 native per-version lock evidence is unknown") from None + if not lock_enabled: + return {"bucket_lock_enabled": False}, False + client = self.adapter.client + request = {"Bucket": self.adapter.bucket, "Key": key, "VersionId": version_id} + try: + legal_response = client.get_object_legal_hold(**request) + retention_response = client.get_object_retention(**request) + except Exception: + raise CloudError("Per-version object-lock evidence is unknown") from None + if not isinstance(legal_response, dict): + raise CloudError("Per-version legal-hold evidence is malformed") + if set(_retention_service_fields(legal_response)) != {"LegalHold"}: + raise CloudError("Per-version legal-hold evidence contains unknown fields") + legal = legal_response.get("LegalHold") + if not isinstance(legal, dict) or legal.get("Status") not in {"ON", "OFF"}: + raise CloudError("Per-version legal-hold evidence is unknown") + if not isinstance(retention_response, dict): + raise CloudError("Per-version retention evidence is malformed") + if set(_retention_service_fields(retention_response)) - {"Retention"}: + raise CloudError("Per-version retention evidence contains unknown fields") + retention = retention_response.get("Retention") + if retention is None: + retention = {} + if not isinstance(retention, dict): + raise CloudError("Per-version retention evidence is unknown") + allowed = {"Mode", "RetainUntilDate", "EventHold", "EventHoldDuration"} + if set(retention) - allowed: + raise CloudError("Per-version retention evidence contains unknown fields") + retain_until = None + if retention: + if "Mode" not in retention or "RetainUntilDate" not in retention: + raise CloudError("Per-version retention evidence is incomplete") + if retention["Mode"] not in {"GOVERNANCE", "COMPLIANCE"}: + raise CloudError("Per-version retention mode is unknown") + value = retention["RetainUntilDate"] + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise CloudError("Per-version retention date is unknown") + retain_until = value.astimezone(timezone.utc) + event_hold = retention.get("EventHold") + if event_hold is not None and event_hold not in {"ON", "OFF"}: + raise CloudError("Per-version event-hold evidence is unknown") + if "EventHoldDuration" in retention: + raise CloudError("Per-version event-hold duration is unsupported") + held = ( + legal["Status"] == "ON" + or event_hold == "ON" + or (retain_until is not None and retain_until > now) + ) + evidence = { + "bucket_lock_enabled": True, + "legal_hold": legal["Status"], + "mode": retention.get("Mode"), + "retain_until": _iso(retain_until) if retain_until is not None else None, + "event_hold": event_hold, + } + return evidence, held + + @staticmethod + def _catalog_record(record: Mapping[str, Any]) -> dict[str, Any]: + return { + "snapshot_id": record["snapshot_id"], + "verified_at": record["verified_at"], + "source_repository": record["source_repository"], + "captured_at": record["captured_at"], + "expires_at": record["expires_at"], + "source_signer": record["source_signer"], + } + + def _protected_versions( + self, + records: Sequence[Mapping[str, Any]], + selected_digests: set[str], + inventory: sqlite3.Connection, + path: Path, + ) -> tuple[sqlite3.Connection, str, int]: + protected = sqlite3.connect(path) + protected.execute( + "CREATE TABLE protected (key TEXT NOT NULL, version_id TEXT NOT NULL, size INTEGER NOT NULL, PRIMARY KEY(key, version_id))" + ) + try: + for record in records: + if record["manifest_sha256"] not in selected_digests: + continue + source = sqlite3.connect(record["index_path"]) + try: + protected.executemany( + """INSERT INTO protected VALUES (?, ?, ?) + ON CONFLICT(key, version_id) DO UPDATE SET + size = CASE WHEN size = excluded.size THEN size ELSE -1 END""", + source.execute( + "SELECT key, version_id, size FROM objects WHERE kind = 'object'" + ), + ) + protected.commit() + finally: + source.close() + inventory.execute("ATTACH DATABASE ? AS protected_manifest", (str(path),)) + try: + unavailable = inventory.execute( + """SELECT 1 FROM protected_manifest.protected AS p + LEFT JOIN versions AS v ON v.key = p.key AND v.version_id = p.version_id + WHERE p.size < 0 OR v.key IS NULL OR v.kind != 'object' OR v.size != p.size + LIMIT 1""" + ).fetchone() + finally: + inventory.execute("DETACH DATABASE protected_manifest") + if unavailable is not None: + raise CloudError("A protected recovery checkpoint version is missing or has changed size") + digest = hashlib.sha256() + count = 0 + for key, version_id, size in protected.execute( + "SELECT key, version_id, size FROM protected ORDER BY key, version_id" + ): + digest.update(_retention_json_bytes({"key": key, "version_id": version_id, "size": size})) + count += 1 + return protected, digest.hexdigest(), count + except BaseException: + protected.close() + raise + + def _plan_inputs( + self, + policy: Mapping[str, Any], + state_dir: Path, + recovery_record_dirs: Sequence[str | os.PathLike[str]], + *, + now: datetime, + temporary: Path, + update_catalog: bool, + maintenance_mode: bool = False, + ) -> dict[str, Any]: + records = self._pin_records(recovery_record_dirs, policy, temporary, now) + catalog, original_catalog_bytes = self._load_catalog(state_dir, policy) + self._merge_catalog_records(catalog, records, policy) + inventory, inventory_digest, version_count = self._inventory( + temporary / "inventory.sqlite3" + ) + try: + if update_catalog: + self._observe_noncurrent(catalog, inventory, now=now) + accepted = catalog["accepted_records"] + selected_digests = _select_retention_records( + [ + { + "manifest_sha256": digest, + "verified_at": entry["verified_at"], + } + for digest, entry in accepted.items() + ], + policy["retention_counts"], + ) + selected_digests.update(policy["known_good_manifest_digests"]) + selected_digests.update(policy["pending_manifest_digests"]) + selected_digests.update( + digest for digest, entry in accepted.items() + if now - _parse_time(entry["verified_at"]) + < timedelta(seconds=policy["grace_seconds"]) + ) + protected, protected_digest, protected_count = self._protected_versions( + records, + selected_digests, + inventory, + temporary / "protected.sqlite3", + ) + try: + evidence = self._bucket_evidence() + first_seen = catalog["first_observed_noncurrent"] + candidates = [] + held_count = 0 + for key, version_id, kind, size, is_latest in inventory.execute( + "SELECT key, version_id, kind, size, is_latest FROM versions ORDER BY key, version_id" + ): + if maintenance_mode: + break + if kind != "object" or is_latest: + continue + if protected.execute( + "SELECT 1 FROM protected WHERE key = ? AND version_id = ?", + (key, version_id), + ).fetchone(): + continue + token = _retention_version_token(key, version_id) + observed = first_seen.get(token) + if not isinstance(observed, str): + raise CloudError("Noncurrent-version grace evidence is missing") + first_time = _parse_time(observed) + if first_time > now: + raise CloudError("Noncurrent-version grace evidence is future-dated") + if (now - first_time).total_seconds() < policy["grace_seconds"]: + continue + lock, held = self._version_lock( + key, + version_id, + lock_enabled=evidence["object_lock_enabled"], + now=now, + ) + lock_digest = _retention_digest(_retention_json_bytes(lock)) + if held: + held_count += 1 + continue + candidates.append( + { + "key": key, + "version_id": version_id, + "size": size, + "first_observed_noncurrent": observed, + "lock_sha256": lock_digest, + "object_id": token, + } + ) + candidate_keys = {candidate["key"] for candidate in candidates} + key_inventory = {} + for key in sorted(candidate_keys): + key_inventory[key] = [ + { + "version_id": version_id, + "kind": kind, + "size": size, + "is_latest": bool(is_latest), + } + for version_id, kind, size, is_latest in inventory.execute( + "SELECT version_id, kind, size, is_latest FROM versions WHERE key = ? ORDER BY version_id", + (key,), + ) + ] + lock_digest = _retention_digest( + _retention_json_bytes( + [ + {"object_id": item["object_id"], "lock_sha256": item["lock_sha256"]} + for item in candidates + ] + ) + ) + if update_catalog: + catalog_raw = _retention_json_bytes(catalog) + catalog_path = self._catalog_path(state_dir) + _write_private_json( + catalog_path, + catalog, + replace=original_catalog_bytes is not None, + ) + else: + catalog_raw = original_catalog_bytes + if catalog_raw is None: + catalog_raw = _retention_json_bytes(catalog) + catalog_digest = _retention_digest(catalog_raw) + manifest_records = sorted( + digest for digest in accepted + ) + protected.close() + return { + "records": records, + "accepted_manifest_digests": manifest_records, + "selected_manifest_digests": sorted(selected_digests), + "inventory": inventory, + "inventory_sha256": inventory_digest, + "version_count": version_count, + "protected_sha256": protected_digest, + "protected_version_count": protected_count, + "catalog": catalog, + "catalog_sha256": catalog_digest, + "evidence": evidence, + "candidates": candidates, + "key_inventory": key_inventory, + "candidate_lock_sha256": lock_digest, + "held_count": held_count, + } + except BaseException: + protected.close() + raise + except BaseException: + inventory.close() + raise + + @staticmethod + def _plans_directory(state_dir: Path, *, create: bool = False) -> Path: + return _ensure_private_directory(state_dir / "retention-plans", create=create) + + @staticmethod + def _operations_directory(state_dir: Path, *, create: bool = False) -> Path: + return _ensure_private_directory(state_dir / "retention-operations", create=create) + + @staticmethod + def _plan_path(state_dir: Path, plan_id: str) -> Path: + if not isinstance(plan_id, str) or not re.fullmatch(r"[0-9a-f]{32}", plan_id): + raise CloudError("Central retention plan ID is invalid") + return _CentralRetention._plans_directory(state_dir) / f"{plan_id}.json" + + @staticmethod + def _journal_path(state_dir: Path, operation_id: str) -> Path: + return _CentralRetention._operations_directory(state_dir, create=True) / f"{operation_id}.json" + + def _read_plan(self, path: Path) -> tuple[dict[str, Any], bytes, str]: + plan, raw = _read_private_json(path) + if set(plan) != _RETENTION_PLAN_FIELDS or type(plan.get("schema_version")) is not int or plan["schema_version"] != 1: + raise CloudError("Central retention plan schema is invalid") + operation_id = plan.get("operation_id") + if not isinstance(operation_id, str) or not re.fullmatch(r"[0-9a-f]{32}", operation_id): + raise CloudError("Central retention plan identity is invalid") + if path.name != f"{operation_id}.json" or path.parent.name != "retention-plans": + raise CloudError("Central retention plan path does not match its identity") + lease_id = plan.get("repository_lease_id") + if not isinstance(lease_id, str) or not re.fullmatch(r"[a-f0-9]{64}", lease_id): + raise CloudError("Central retention plan repository lease identity is invalid") + _repository_identity(plan.get("target", {}).get("repository_id") if isinstance(plan.get("target"), dict) else None) + if plan.get("target") != { + "endpoint_url": self.adapter.endpoint_url, + "bucket": self.adapter.bucket, + "prefix": self.adapter.prefix, + "repository_id": plan.get("target", {}).get("repository_id") if isinstance(plan.get("target"), dict) else None, + }: + raise CloudError("Central retention plan target is invalid") + for name in ( + "policy_sha256", + "grant_sha256", + "inventory_sha256", + "catalog_sha256", + "protected_sha256", + "candidate_lock_sha256", + ): + if not isinstance(plan.get(name), str) or not _FULL_ID.fullmatch(plan[name]): + raise CloudError("Central retention plan digest is invalid") + evidence = plan.get("bucket_evidence_sha256") + if not isinstance(evidence, dict) or set(evidence) != { + "versioning_sha256", "lifecycle_sha256", "object_lock_sha256" + } or any(not isinstance(value, str) or not _FULL_ID.fullmatch(value) for value in evidence.values()): + raise CloudError("Central retention bucket evidence is invalid") + for name in ("created_at", "deadline", "grant_expires_at"): + _parse_time(plan.get(name)) + if type(plan.get("version_count")) is not int or plan["version_count"] < 0: + raise CloudError("Central retention plan inventory count is invalid") + if type(plan.get("object_lock_enabled")) is not bool: + raise CloudError("Central retention plan object-lock state is invalid") + limits = plan.get("apply_limits") + if not isinstance(limits, dict) or set(limits) != {"max_apply_objects", "max_apply_bytes"} or any( + type(value) is not int or value < 1 for value in limits.values() + ): + raise CloudError("Central retention plan limits are invalid") + candidates = plan.get("candidates") + if not isinstance(candidates, list): + raise CloudError("Central retention plan candidates are invalid") + for candidate in candidates: + if not isinstance(candidate, dict) or set(candidate) != { + "key", "version_id", "size", "first_observed_noncurrent", "lock_sha256", "object_id" + }: + raise CloudError("Central retention plan candidate is invalid") + _relative_key(candidate.get("key"), self.adapter.prefix) + if not isinstance(candidate.get("version_id"), str) or not candidate["version_id"]: + raise CloudError("Central retention plan version ID is invalid") + if type(candidate.get("size")) is not int or candidate["size"] < 0: + raise CloudError("Central retention plan candidate size is invalid") + _parse_time(candidate.get("first_observed_noncurrent")) + if not isinstance(candidate.get("lock_sha256"), str) or not _FULL_ID.fullmatch(candidate["lock_sha256"]): + raise CloudError("Central retention plan candidate lock digest is invalid") + if candidate.get("object_id") != _retention_version_token(candidate["key"], candidate["version_id"]): + raise CloudError("Central retention plan candidate identity is invalid") + key_inventory = plan.get("key_inventory") + if not isinstance(key_inventory, dict): + raise CloudError("Central retention plan key inventory is invalid") + for key, items in key_inventory.items(): + _relative_key(key, self.adapter.prefix) + if not isinstance(items, list): + raise CloudError("Central retention plan key inventory is invalid") + for item in items: + if not isinstance(item, dict) or set(item) != {"version_id", "kind", "size", "is_latest"}: + raise CloudError("Central retention plan key inventory is invalid") + if not isinstance(item["version_id"], str) or not item["version_id"]: + raise CloudError("Central retention plan key inventory is invalid") + if item["kind"] not in {"object", "delete_marker"} or type(item["is_latest"]) is not bool: + raise CloudError("Central retention plan key inventory is invalid") + if item["kind"] == "object" and (type(item["size"]) is not int or item["size"] < 0): + raise CloudError("Central retention plan key inventory is invalid") + if item["kind"] == "delete_marker" and item["size"] is not None: + raise CloudError("Central retention plan key inventory is invalid") + return plan, raw, _retention_digest(raw) + + @staticmethod + def _operation_ledger(ledger: Any | None) -> Any: + if ledger is None or not callable(getattr(ledger, "begin", None)) or not callable( + getattr(ledger, "reconcile", None) + ): + raise CloudError("An existing repository operation ledger is required") + return ledger + + def _list_key_versions(self, key: str) -> list[dict[str, Any]]: + _relative_key(key, self.adapter.prefix) + versions = [] + key_marker = None + version_marker = None + seen_pages = set() + seen_versions = set() + while True: + marker = (key_marker or "", version_marker or "") + if marker in seen_pages: + raise CloudError("Exact-key version pagination repeated a marker") + seen_pages.add(marker) + request = {"Bucket": self.adapter.bucket, "Prefix": key, "MaxKeys": self.adapter.page_size} + if key_marker is not None: + request["KeyMarker"] = key_marker + if version_marker is not None: + request["VersionIdMarker"] = version_marker + try: + response = self.adapter.client.list_object_versions(**request) + except Exception: + raise CloudError("Could not recheck the exact object version inventory") from None + if not isinstance(response, dict): + raise CloudError("Exact-key version inventory is malformed") + for collection, kind in (("Versions", "object"), ("DeleteMarkers", "delete_marker")): + items = response.get(collection, []) + if not isinstance(items, list): + raise CloudError("Exact-key version inventory is malformed") + for item in items: + if not isinstance(item, dict) or type(item.get("IsLatest")) is not bool: + raise CloudError("Exact-key version inventory is malformed") + item_key = item.get("Key") + _relative_key(item_key, self.adapter.prefix) + version_id = item.get("VersionId") + if not isinstance(version_id, str) or not version_id: + raise CloudError("Exact-key version ID is missing") + if item_key != key: + continue + token = (item_key, version_id) + if token in seen_versions: + raise CloudError("Exact-key version inventory contains duplicates") + seen_versions.add(token) + size = item.get("Size") if kind == "object" else None + if kind == "object" and (type(size) is not int or size < 0): + raise CloudError("Exact-key object version size is invalid") + versions.append({ + "version_id": version_id, + "kind": kind, + "size": size, + "is_latest": item["IsLatest"], + }) + truncated = response.get("IsTruncated", False) + if type(truncated) is not bool: + raise CloudError("Exact-key version pagination state is invalid") + if not truncated: + break + next_key = response.get("NextKeyMarker") + next_version = response.get("NextVersionIdMarker") + if not isinstance(next_key, str) or not next_key: + raise CloudError("Truncated exact-key inventory lacks a key marker") + if next_version is not None and not isinstance(next_version, str): + raise CloudError("Exact-key version marker is invalid") + next_marker = (next_key, next_version or "") + if next_marker == marker: + raise CloudError("Exact-key version pagination did not progress") + key_marker, version_marker = next_key, next_version + if versions and sum(item["is_latest"] for item in versions) != 1: + raise CloudError("Exact-key inventory has an invalid latest-version set") + return sorted(versions, key=lambda item: item["version_id"]) + + def _single_attempt_mutation_client(self) -> Any: + try: + import boto3 + from botocore.config import Config + except ImportError: + raise CloudError("boto3 is required for central retention deletion") from None + options = { + "endpoint_url": self.adapter.endpoint_url, + "config": Config( + connect_timeout=10, + read_timeout=30, + retries={"total_max_attempts": 1, "mode": "standard"}, + signature_version="s3v4", + ), + } + if self.adapter.region_name is not None: + options["region_name"] = self.adapter.region_name + if self.adapter._credential_pair is not None: + options["aws_access_key_id"], options["aws_secret_access_key"] = self.adapter._credential_pair + try: + return boto3.client("s3", **options) + except Exception: + raise CloudError("Could not create a single-attempt retention client") from None + + @staticmethod + def _require_single_attempt_client(client: Any) -> None: + meta = getattr(client, "meta", None) + if meta is None: + return + config = getattr(meta, "config", None) + retries = getattr(config, "retries", None) + if not isinstance(retries, dict) or retries.get("total_max_attempts") != 1: + raise CloudError("Retention mutation clients must be configured for one attempt") + + @staticmethod + def _journal_value(plan: Mapping[str, Any], plan_sha256: str) -> dict[str, Any]: + return { + "schema_version": 1, + "operation_id": plan["operation_id"], + "plan_sha256": plan_sha256, + "status": "prepared", + "updated_at": _iso(datetime.now(timezone.utc)), + "items": [ + {"object_id": item["object_id"], "status": "pending"} + for item in plan["candidates"] + ], + } + + @staticmethod + def _save_journal(path: Path, journal: dict[str, Any]) -> None: + journal["updated_at"] = _iso(datetime.now(timezone.utc)) + _write_private_json(path, journal, replace=os.path.lexists(path)) + + def plan( + self, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + recovery_record_dirs: Sequence[str | os.PathLike[str]], + *, + quiescent_window_confirmed: bool, + repository_lease_id: str, + ) -> dict[str, Any]: + if quiescent_window_confirmed is not True: + raise CloudError("Central retention requires an explicitly coordinated quiescent window") + if not isinstance(repository_lease_id, str) or not re.fullmatch(r"[a-f0-9]{64}", repository_lease_id): + raise CloudError("Central retention repository lease identity is invalid") + authority = self._authority(policy_path, grant_path) + policy, grant, state_dir, now, policy_digest, grant_digest = authority[:6] + with _retention_catalog_lock(state_dir): + authority = self._authority(policy_path, grant_path) + policy, grant, state_dir, now, policy_digest, grant_digest = authority[:6] + with tempfile.TemporaryDirectory(prefix="retention-plan-", dir=state_dir) as temporary_name: + temporary = Path(temporary_name) + os.chmod(temporary, 0o700) + inputs = self._plan_inputs( + policy, + state_dir, + recovery_record_dirs, + now=now, + temporary=temporary, + update_catalog=True, + ) + candidates = inputs["candidates"] + total_bytes = sum(item["size"] for item in candidates) + limits = { + "max_apply_objects": min(policy["max_apply_objects"], grant["max_apply_objects"]), + "max_apply_bytes": min(policy["max_apply_bytes"], grant["max_apply_bytes"]), + } + if len(candidates) > limits["max_apply_objects"] or total_bytes > limits["max_apply_bytes"]: + raise CloudError("Central retention candidates exceed the authorized apply bounds") + operation_id = uuid.uuid4().hex + created = datetime.now(timezone.utc) + deadline = min( + created + timedelta(seconds=policy["plan_max_age_seconds"]), + _parse_time(grant["expires_at"]), + ) + plan = { + "schema_version": 1, + "operation_id": operation_id, + "created_at": _iso(created), + "deadline": _iso(deadline), + "target": _retention_target(policy), + "repository_lease_id": repository_lease_id, + "policy_id": policy["policy_id"], + "policy_revision": policy["revision"], + "policy_sha256": policy_digest, + "grant_sha256": grant_digest, + "grant_expires_at": _iso(_parse_time(grant["expires_at"])), + "apply_limits": limits, + "inventory_sha256": inputs["inventory_sha256"], + "version_count": inputs["version_count"], + "catalog_sha256": inputs["catalog_sha256"], + "accepted_manifest_digests": inputs["accepted_manifest_digests"], + "selected_manifest_digests": inputs["selected_manifest_digests"], + "protected_sha256": inputs["protected_sha256"], + "protected_version_count": inputs["protected_version_count"], + "candidate_lock_sha256": inputs["candidate_lock_sha256"], + "held_count": inputs["held_count"], + "bucket_evidence_sha256": inputs["evidence"]["digests"], + "object_lock_enabled": inputs["evidence"]["object_lock_enabled"], + "candidates": candidates, + "key_inventory": inputs["key_inventory"], + } + plan_path = self._plans_directory(state_dir, create=True) / f"{operation_id}.json" + raw = _write_immutable_plan(plan_path, plan) + return { + "status": "planned", + "operation_id": operation_id, + "plan_sha256": _retention_digest(raw), + "candidate_count": len(candidates), + "candidate_bytes": total_bytes, + "protected_version_count": inputs["protected_version_count"], + "held_version_count": inputs["held_count"], + "inventory_version_count": inputs["version_count"], + "deadline": plan["deadline"], + } + + def apply( + self, + plan_id: str, + expected_plan_sha256: str, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + recovery_record_dirs: Sequence[str | os.PathLike[str]], + *, + quiescent_window_confirmed: bool, + ledger: Any, + cancel_event: Event | None = None, + mutation_client: Any | None = None, + expected_repository_lease_id: str | None = None, + ) -> dict[str, Any]: + if quiescent_window_confirmed is not True: + raise CloudError("Central retention requires an explicitly coordinated quiescent window") + if not isinstance(expected_plan_sha256, str) or not _FULL_ID.fullmatch(expected_plan_sha256): + raise CloudError("Central retention expected plan digest is invalid") + authority = self._authority(policy_path, grant_path) + policy, grant, state_dir, _now, policy_digest, grant_digest = authority[:6] + with _retention_catalog_lock(state_dir): + policy, grant, state_dir, now, policy_digest, grant_digest = self._authority( + policy_path, grant_path + )[:6] + plan_path = self._plan_path(state_dir, plan_id) + plan, plan_raw, plan_digest = self._read_plan(plan_path) + if ( + expected_repository_lease_id is not None + and plan["repository_lease_id"] != expected_repository_lease_id + ): + raise CloudError("Central retention plan belongs to a different repository lease") + if plan_digest != expected_plan_sha256: + raise CloudError("Central retention plan digest differs from the reviewed plan") + if plan["operation_id"] != plan_id or plan["target"] != _retention_target(policy): + raise CloudError("Central retention plan does not match the current target") + if plan["policy_id"] != policy["policy_id"] or plan["policy_revision"] != policy["revision"]: + raise CloudError("Central retention policy differs from the reviewed plan") + if policy_digest != plan["policy_sha256"] or grant_digest != plan["grant_sha256"]: + raise CloudError("Central retention authority changed after planning") + if now >= _parse_time(plan["deadline"]): + raise CloudError("Central retention plan has expired") + if now >= _parse_time(plan["grant_expires_at"]): + raise CloudError("Central retention grant has expired") + if len(plan["candidates"]) > plan["apply_limits"]["max_apply_objects"] or sum( + item["size"] for item in plan["candidates"] + ) > plan["apply_limits"]["max_apply_bytes"]: + raise CloudError("Central retention plan exceeds its authorized apply bounds") + journal_path = self._journal_path(state_dir, plan_id) + if os.path.lexists(journal_path): + raise CloudError("Central retention plan was already applied or has an unresolved attempt") + from .operations import OperationPlan, OperationStatus + + active_ledger = self._operation_ledger(ledger) + operation_plan = OperationPlan( + operation_id=plan_id, + repository_id=plan["repository_lease_id"], + label="central-retention", + argv=("delete_noncurrent_versions",), + destructive=True, + ) + lease = active_ledger.begin(operation_plan) + try: + with tempfile.TemporaryDirectory(prefix="retention-apply-", dir=state_dir) as temporary_name: + temporary = Path(temporary_name) + os.chmod(temporary, 0o700) + current_inputs = self._plan_inputs( + policy, + state_dir, + recovery_record_dirs, + now=now, + temporary=temporary, + update_catalog=False, + ) + comparisons = ( + ("inventory_sha256", current_inputs["inventory_sha256"]), + ("version_count", current_inputs["version_count"]), + ("catalog_sha256", current_inputs["catalog_sha256"]), + ("accepted_manifest_digests", current_inputs["accepted_manifest_digests"]), + ("selected_manifest_digests", current_inputs["selected_manifest_digests"]), + ("protected_sha256", current_inputs["protected_sha256"]), + ("protected_version_count", current_inputs["protected_version_count"]), + ("candidate_lock_sha256", current_inputs["candidate_lock_sha256"]), + ("candidates", current_inputs["candidates"]), + ("key_inventory", current_inputs["key_inventory"]), + ("bucket_evidence_sha256", current_inputs["evidence"]["digests"]), + ("object_lock_enabled", current_inputs["evidence"]["object_lock_enabled"]), + ) + if any(plan[name] != value for name, value in comparisons): + raise CloudError("Central retention inputs drifted after planning") + + self._assert_live_authority(plan, policy_path, grant_path) + mutation = mutation_client if mutation_client is not None else self._single_attempt_mutation_client() + self._require_single_attempt_client(mutation) + journal_path = self._journal_path(state_dir, plan_id) + if os.path.lexists(journal_path): + raise CloudError("Central retention plan was already applied or has an unresolved attempt") + journal = self._journal_value(plan, plan_digest) + self._save_journal(journal_path, journal) + except BaseException: + try: + lease.finish(OperationStatus.FAILED, 2) + finally: + lease.close() + raise + expected_by_key = {key: [dict(item) for item in items] for key, items in plan["key_inventory"].items()} + deleted_count = 0 + deleted_bytes = 0 + + def cancelled_at(index: int, *, after_intent: bool = False) -> dict[str, Any] | None: + if cancel_event is None or not cancel_event.is_set(): + return None + if after_intent: + journal["items"][index]["status"] = "pending" + journal["status"] = "cancelled" + self._save_journal(journal_path, journal) + lease.finish(OperationStatus.CANCELLED, 130) + return { + "status": "cancelled", + "operation_id": plan_id, + "deleted_count": deleted_count, + "deleted_bytes": deleted_bytes, + "remaining_count": len(plan["candidates"]) - deleted_count, + } + + try: + for index, candidate in enumerate(plan["candidates"]): + cancelled = cancelled_at(index) + if cancelled is not None: + return cancelled + try: + self._assert_live_authority(plan, policy_path, grant_path) + except CloudError: + journal["status"] = "stopped" + self._save_journal(journal_path, journal) + lease.finish(OperationStatus.FAILED, 2) + return { + "status": "stopped", + "operation_id": plan_id, + "deleted_count": deleted_count, + "deleted_bytes": deleted_bytes, + "remaining_count": len(plan["candidates"]) - deleted_count, + } + expected_versions = expected_by_key.get(candidate["key"], []) + try: + current_versions = self._list_key_versions(candidate["key"]) + candidate_record = next( + item for item in expected_versions if item["version_id"] == candidate["version_id"] + ) + except (CloudError, StopIteration): + current_versions = [] + candidate_record = None + if ( + current_versions != expected_versions + or candidate_record is None + or candidate_record["kind"] != "object" + or candidate_record["is_latest"] + ): + journal["status"] = "stopped" + self._save_journal(journal_path, journal) + lease.finish(OperationStatus.FAILED, 2) + return { + "status": "stopped", + "operation_id": plan_id, + "deleted_count": deleted_count, + "deleted_bytes": deleted_bytes, + "remaining_count": len(plan["candidates"]) - deleted_count, + } + try: + lock, held = self._version_lock( + candidate["key"], + candidate["version_id"], + lock_enabled=plan["object_lock_enabled"], + now=datetime.now(timezone.utc), + ) + except CloudError: + lock, held = {}, True + try: + self._assert_live_authority(plan, policy_path, grant_path) + except CloudError: + held = True + if held or _retention_digest(_retention_json_bytes(lock)) != candidate["lock_sha256"]: + journal["status"] = "stopped" + self._save_journal(journal_path, journal) + lease.finish(OperationStatus.FAILED, 2) + return { + "status": "stopped", + "operation_id": plan_id, + "deleted_count": deleted_count, + "deleted_bytes": deleted_bytes, + "remaining_count": len(plan["candidates"]) - deleted_count, + } + cancelled = cancelled_at(index) + if cancelled is not None: + return cancelled + journal["items"][index]["status"] = "intent" + self._save_journal(journal_path, journal) + try: + self._assert_live_authority(plan, policy_path, grant_path) + except CloudError: + journal["items"][index]["status"] = "pending" + journal["status"] = "stopped" + self._save_journal(journal_path, journal) + lease.finish(OperationStatus.FAILED, 2) + return { + "status": "stopped", + "operation_id": plan_id, + "deleted_count": deleted_count, + "deleted_bytes": deleted_bytes, + "remaining_count": len(plan["candidates"]) - deleted_count, + } + cancelled = cancelled_at(index, after_intent=True) + if cancelled is not None: + return cancelled + try: + response = mutation.delete_object( + Bucket=self.adapter.bucket, + Key=candidate["key"], + VersionId=candidate["version_id"], + ) + if not isinstance(response, dict) or response.get("VersionId") != candidate["version_id"]: + raise CloudError("Retention delete response did not confirm the exact version") + remaining_versions = self._list_key_versions(candidate["key"]) + if any(item["version_id"] == candidate["version_id"] for item in remaining_versions): + raise CloudError("Retention delete response could not be verified") + except Exception: + journal["items"][index]["status"] = "uncertain" + journal["status"] = "uncertain" + self._save_journal(journal_path, journal) + return { + "status": "uncertain", + "operation_id": plan_id, + "deleted_count": deleted_count, + "deleted_bytes": deleted_bytes, + "uncertain_count": 1, + "remaining_count": len(plan["candidates"]) - deleted_count, + } + journal["items"][index]["status"] = "confirmed" + self._save_journal(journal_path, journal) + expected_by_key[candidate["key"]] = [ + item for item in expected_versions if item["version_id"] != candidate["version_id"] + ] + deleted_count += 1 + deleted_bytes += candidate["size"] + journal["status"] = "complete" + self._save_journal(journal_path, journal) + lease.finish(OperationStatus.SUCCEEDED, 0) + return { + "status": "complete", + "operation_id": plan_id, + "deleted_count": deleted_count, + "deleted_bytes": deleted_bytes, + "remaining_count": 0, + } + finally: + lease.close() + + def reconcile( + self, + plan_path: str | os.PathLike[str], + *, + ledger: Any, + expected_repository_lease_id: str | None = None, + ) -> dict[str, Any]: + path = Path(plan_path).absolute() + _ensure_private_directory(path.parent) + plan, _raw, _digest = self._read_plan(path) + state_dir = _ensure_private_directory(path.parent.parent) + if path.parent != self._plans_directory(state_dir): + raise CloudError("Central retention plan is outside its private state directory") + with _retention_catalog_lock(state_dir): + return self._reconcile_locked( + path, ledger=ledger, + expected_repository_lease_id=expected_repository_lease_id, + ) + + def _reconcile_locked( + self, + path: Path, + *, + ledger: Any, + expected_repository_lease_id: str | None = None, + ) -> dict[str, Any]: + plan, _plan_raw, plan_digest = self._read_plan(path) + if ( + expected_repository_lease_id is not None + and plan["repository_lease_id"] != expected_repository_lease_id + ): + raise CloudError("Central retention plan belongs to a different repository lease") + state_dir = _ensure_private_directory(path.parent.parent) + journal_path = self._journal_path(state_dir, plan["operation_id"]) + journal, _journal_raw = _read_private_json(journal_path) + if set(journal) != _RETENTION_JOURNAL_FIELDS or journal.get("schema_version") != 1: + raise CloudError("Central retention operation journal schema is invalid") + if journal.get("operation_id") != plan["operation_id"] or journal.get("plan_sha256") != plan_digest: + raise CloudError("Central retention operation journal does not match its plan") + if not isinstance(journal.get("items"), list) or len(journal["items"]) != len(plan["candidates"]): + raise CloudError("Central retention operation journal items are invalid") + observed = [] + for candidate, item in zip(plan["candidates"], journal["items"], strict=True): + if not isinstance(item, dict) or set(item) != {"object_id", "status"} or item["object_id"] != candidate["object_id"]: + raise CloudError("Central retention operation journal items are invalid") + try: + versions = self._list_key_versions(candidate["key"]) + except CloudError: + observed.append("unknown") + continue + present = any(version["version_id"] == candidate["version_id"] for version in versions) + observed.append("present" if present else "absent") + counts = {status: observed.count(status) for status in ("absent", "present", "unknown")} + journal["items"] = [ + {"object_id": item["object_id"], "status": status} + for item, status in zip(journal["items"], observed, strict=True) + ] + if counts["unknown"]: + journal["status"] = "uncertain" + outcome = None + public_status = "uncertain" + elif counts["present"]: + journal["status"] = "reconciled_abandoned" + from .operations import ReconciliationOutcome + + outcome = ReconciliationOutcome.ABANDONED + public_status = "reconciled_abandoned" + else: + journal["status"] = "reconciled_succeeded" + from .operations import ReconciliationOutcome + + outcome = ReconciliationOutcome.SUCCEEDED + public_status = "reconciled_succeeded" + self._save_journal(journal_path, journal) + if outcome is not None: + try: + active_ledger = self._operation_ledger(ledger) + active_ledger.reconcile( + plan["repository_lease_id"], + plan["operation_id"], + outcome, + ) + except (ValueError, KeyError): + pass + return { + "status": public_status, + "operation_id": plan["operation_id"], + "absent_count": counts["absent"], + "present_count": counts["present"], + "unknown_count": counts["unknown"], + } + + +class _CentralMaintenance: + """Reviewed logical maintenance over the current Restic repository namespace.""" + + def __init__( + self, + adapter: S3CloudAdapter, + policy_path: str | os.PathLike[str], + grant_path: str | os.PathLike[str], + record_dirs: Sequence[str | os.PathLike[str]], + *, + quiescent_window_confirmed: bool, + repository_lease_id: str, + reconcile_only: bool = False, + ) -> None: + self.adapter = adapter + self.owner = _CentralRetention(adapter) + self.policy_path = Path(policy_path).absolute() + self.grant_path = Path(grant_path).absolute() + if isinstance(record_dirs, (str, bytes)) or not isinstance(record_dirs, Sequence): + raise CloudError("Central maintenance recovery record inputs are invalid") + self.record_dirs = tuple(record_dirs) + if quiescent_window_confirmed is not True: + raise CloudError("Central maintenance requires an explicitly coordinated quiescent window") + if not isinstance(repository_lease_id, str) or not re.fullmatch( + r"[a-f0-9]{64}", repository_lease_id + ): + raise CloudError("Central maintenance repository lease identity is invalid") + self.repository_lease_id = repository_lease_id + self.reconcile_only = reconcile_only + self._lock_context = None + self._authority_snapshot: tuple[str, str, Path] | None = None + self._selection_data: dict[str, Any] | None = None + self._last_verified_protected: dict[str, Any] | None = None + self.selection: CloudMaintenanceSelection | None = None + + def __enter__(self) -> _CentralMaintenance: + authority = self.owner._authority( + self.policy_path, + self.grant_path, + expected_action="maintain_current_repository", + allow_expired=self.reconcile_only, + ) + state_dir = authority[2] + self._lock_context = _retention_catalog_lock(state_dir) + self._lock_context.__enter__() + try: + self._set_authority_snapshot() + captured = self._capture(allow_expired=self.reconcile_only) + self._selection_data = captured + self.selection = CloudMaintenanceSelection( + accepted_snapshot_ids=tuple(captured["accepted_snapshot_ids"]), + protected_snapshot_ids=tuple(captured["protected_snapshot_ids"]), + protected_version_sha256=captured["protected_sha256"], + policy_sha256=captured["policy_sha256"], + grant_sha256=captured["grant_sha256"], + catalog_sha256=captured["catalog_sha256"], + inventory_sha256=captured["inventory_sha256"], + resources=MappingProxyType( + { + **captured["resources"], + "resource_budget": MappingProxyType( + dict(captured["resources"]["resource_budget"]) + ), + } + ), + ) + return self + except BaseException: + self._release_lock() + raise + + def __exit__(self, exc_type, exc, traceback) -> None: + self._release_lock() + + def _release_lock(self) -> None: + if self._lock_context is not None: + context, self._lock_context = self._lock_context, None + context.__exit__(None, None, None) + + def _set_authority_snapshot(self) -> None: + authority = self.owner._authority( + self.policy_path, + self.grant_path, + expected_action="maintain_current_repository", + allow_expired=self.reconcile_only, + ) + self._authority_snapshot = (authority[4], authority[5], authority[2]) + + def _authority( + self, *, allow_expired: bool = False + ) -> tuple[dict[str, Any], dict[str, Any], Path, datetime, str, str, Path, Path]: + authority = self.owner._authority( + self.policy_path, + self.grant_path, + expected_action="maintain_current_repository", + allow_expired=allow_expired, + ) + if self._authority_snapshot is not None and ( + authority[4], authority[5], authority[2] + ) != self._authority_snapshot: + raise CloudError("Central maintenance authority changed during the session") + return authority + + def _maintenance_resources( + self, + inputs: Mapping[str, Any], + policy: Mapping[str, Any], + grant: Mapping[str, Any], + *, + now: datetime, + ) -> tuple[dict[str, Any], str, str]: + inventory: sqlite3.Connection = inputs["inventory"] + evidence = inputs["evidence"] + maximum_objects = min(policy["max_apply_objects"], grant["max_apply_objects"]) + maximum_bytes = min(policy["max_apply_bytes"], grant["max_apply_bytes"]) + current_payload_count = 0 + current_payload_bytes = 0 + inventory_count = 0 + inventory_digest = hashlib.sha256() + resource_digest = hashlib.sha256() + for key, version_id, kind, size, is_latest in inventory.execute( + "SELECT key, version_id, kind, size, is_latest FROM versions ORDER BY key, version_id" + ): + if not is_latest or _is_lock_key(_relative_key(key, self.adapter.prefix)): + continue + inventory_count += 1 + inventory_item = { + "key": key, + "version_id": version_id, + "kind": kind, + "size": size, + "is_latest": True, + } + inventory_digest.update(_retention_json_bytes(inventory_item)) + if kind == "object": + lock, held = self.owner._version_lock( + key, + version_id, + lock_enabled=evidence["object_lock_enabled"], + now=now, + ) + lock_sha256 = _retention_digest(_retention_json_bytes(lock)) + current_payload_count += 1 + current_payload_bytes += size + if current_payload_count > maximum_objects or current_payload_bytes > maximum_bytes: + raise CloudError( + "Central maintenance current payload inventory exceeds authorized bounds" + ) + else: + held = False + lock_sha256 = None + resource_digest.update( + _retention_json_bytes( + { + **inventory_item, + "held": held, + "lock_sha256": lock_sha256, + } + ) + ) + maintenance = policy["maintenance"] + resources = { + "inventory_version_count": inventory_count, + "current_payload_count": current_payload_count, + "current_payload_bytes": current_payload_bytes, + "max_apply_objects": maximum_objects, + "max_apply_bytes": maximum_bytes, + "resource_budget": dict(maintenance["resource_budget"]), + "timeout_seconds": maintenance["timeout_seconds"], + } + return ( + resources, + inventory_digest.hexdigest(), + resource_digest.hexdigest(), + ) + + def _capture(self, *, allow_expired: bool = False) -> dict[str, Any]: + policy, grant, state_dir, now, policy_sha256, grant_sha256 = self._authority( + allow_expired=allow_expired + )[:6] + with tempfile.TemporaryDirectory(prefix="maintenance-plan-", dir=state_dir) as temporary_name: + temporary = Path(temporary_name) + os.chmod(temporary, 0o700) + original_catalog, original_catalog_raw = self.owner._load_catalog( + state_dir, policy + ) + original_accepted_digests = set(original_catalog["accepted_records"]) + inputs = self.owner._plan_inputs( + policy, + state_dir, + self.record_dirs, + now=now, + temporary=temporary, + update_catalog=False, + maintenance_mode=True, + ) + try: + supplied_accepted_digests = { + record["manifest_sha256"] for record in inputs["records"] + } + if supplied_accepted_digests != original_accepted_digests: + raise CloudError( + "Logical maintenance cannot enroll or omit accepted recovery records" + ) + if inputs["catalog_sha256"] != _retention_digest(original_catalog_raw): + raise CloudError( + "Logical maintenance cannot change central catalog acceptance state" + ) + resources, maintenance_inventory_sha256, resource_inventory_sha256 = self._maintenance_resources( + inputs, policy, grant, now=now + ) + catalog = inputs["catalog"] + accepted_snapshot_ids = sorted( + entry["snapshot_id"] + for entry in catalog["accepted_records"].values() + ) + if len(set(accepted_snapshot_ids)) != len(accepted_snapshot_ids): + raise CloudError("Central recovery catalog repeats a snapshot identity") + selected = set(inputs["selected_manifest_digests"]) + protected_snapshot_ids = sorted( + entry["snapshot_id"] + for digest, entry in catalog["accepted_records"].items() + if digest in selected + ) + data = { + "policy": policy, + "grant": grant, + "state_dir": state_dir, + "policy_sha256": policy_sha256, + "grant_sha256": grant_sha256, + "catalog_sha256": inputs["catalog_sha256"], + "accepted_manifest_digests": inputs["accepted_manifest_digests"], + "selected_manifest_digests": inputs["selected_manifest_digests"], + "accepted_snapshot_ids": accepted_snapshot_ids, + "protected_snapshot_ids": protected_snapshot_ids, + "protected_sha256": inputs["protected_sha256"], + "protected_version_count": inputs["protected_version_count"], + "inventory_sha256": maintenance_inventory_sha256, + "version_count": resources["inventory_version_count"], + "resource_inventory_sha256": resource_inventory_sha256, + "resources": resources, + "evidence": inputs["evidence"], + } + return data + finally: + inputs["inventory"].close() + + @staticmethod + def _compare_capture(expected: Mapping[str, Any], actual: Mapping[str, Any]) -> None: + for name in ( + "policy_sha256", + "grant_sha256", + "catalog_sha256", + "accepted_manifest_digests", + "selected_manifest_digests", + "accepted_snapshot_ids", + "protected_snapshot_ids", + "protected_sha256", + "protected_version_count", + "inventory_sha256", + "version_count", + "resource_inventory_sha256", + "resources", + ): + if expected[name] != actual[name]: + raise CloudError("Central maintenance inputs drifted after selection") + + @staticmethod + def _plans_directory(state_dir: Path, *, create: bool = False) -> Path: + return _CentralRetention._plans_directory(state_dir, create=create) + + @staticmethod + def _operations_directory(state_dir: Path, *, create: bool = False) -> Path: + return _CentralRetention._operations_directory(state_dir, create=create) + + def _plan_path(self, plan_id: str) -> Path: + if not isinstance(plan_id, str) or not re.fullmatch(r"[0-9a-f]{32}", plan_id): + raise CloudError("Central maintenance plan ID is invalid") + if self._selection_data is None: + raise CloudError("Central maintenance session is not active") + return self._plans_directory(self._selection_data["state_dir"]) / f"{plan_id}.json" + + def _journal_path(self, plan_id: str) -> Path: + if self._selection_data is None: + raise CloudError("Central maintenance session is not active") + return self._operations_directory( + self._selection_data["state_dir"], create=True + ) / f"{plan_id}.json" + + def _assert_no_unresolved_maintenance(self, state_dir: Path) -> None: + operations_path = state_dir / "retention-operations" + if not os.path.lexists(operations_path): + return + operations = self._operations_directory(state_dir) + for path in sorted(operations.glob("*.json")): + journal, _raw = _read_private_json(path) + if journal.get("operation_kind") != "maintenance": + continue + try: + plan_id = journal["operation_id"] + plan_sha256 = journal["plan_sha256"] + if path.name != f"{plan_id}.json": + raise CloudError("Central maintenance journal path is invalid") + plan, _digest = self._private_plan(plan_id, plan_sha256) + journal, _path = self._read_journal(plan_id, plan_sha256) + phases = journal["phases"] + previous_complete = True + for phase in _MAINTENANCE_PHASES: + value = phases[phase] + if value is not None and not previous_complete: + raise CloudError("Central maintenance journal phase order is invalid") + if value is not None and value["status"] == "completed": + if phase == "protected_versions": + if value["evidence"] != { + "protected_version_sha256": plan["protected_sha256"], + "protected_version_count": plan["protected_version_count"], + }: + raise CloudError("Central maintenance protected phase is invalid") + elif self._phase_evidence(plan, phase, value["evidence"]) != value["evidence"]: + raise CloudError("Central maintenance phase evidence is invalid") + previous_complete = value is not None and value["status"] == "completed" + mutation_started = self._mutation_started(journal) + status = journal["status"] + completed = all( + phases[phase] is not None and phases[phase]["status"] == "completed" + for phase in _MAINTENANCE_PHASES + ) + terminal = status in {"complete", "partial_abandoned", "abandoned"} + if terminal: + reconciliation = journal["reconciliation"] + if not isinstance(reconciliation, dict) or set(reconciliation) != { + "outcome", "repository_health", "snapshot_inventory_sha256", + "visible_snapshot_ids", "protected_version_sha256", + "protected_version_count", + }: + raise CloudError("Central maintenance terminal evidence is invalid") + checked = self._health_observation({ + "repository_health": reconciliation["repository_health"], + "config_repository_id": plan["config_repository_id"], + "visible_snapshot_ids": reconciliation["visible_snapshot_ids"], + "snapshot_inventory_sha256": reconciliation["snapshot_inventory_sha256"], + }, plan, label="terminal") + visible = set(checked["visible_snapshot_ids"]) + initial = set(plan["visible_snapshot_ids"]) + expected_survivors = initial - set(plan["target_snapshot_ids"]) + if (not expected_survivors <= visible or visible - initial + or not set(plan["protected_snapshot_ids"]) <= visible + or reconciliation["protected_version_sha256"] != plan["protected_sha256"] + or reconciliation["protected_version_count"] != plan["protected_version_count"] + or (status == "complete") != completed + or (status == "partial_abandoned") != (mutation_started and not completed) + or (status == "abandoned") != (not mutation_started) + or reconciliation["outcome"] != ( + "succeeded" if status == "complete" else "abandoned" + ) + or (completed and visible != expected_survivors) + or (completed and phases["protected_versions"]["evidence"] != { + "protected_version_sha256": reconciliation["protected_version_sha256"], + "protected_version_count": reconciliation["protected_version_count"], + })): + raise CloudError("Central maintenance terminal evidence is inconsistent") + except (CloudError, KeyError, TypeError, ValueError): + raise CloudError("An existing central maintenance journal is invalid") from None + if not terminal: + raise CloudError("An earlier central maintenance attempt must be reconciled before planning") + + def _read_plan(self, path: Path) -> tuple[dict[str, Any], bytes, str]: + plan, raw = _read_private_json(path) + if ( + set(plan) != _MAINTENANCE_PLAN_FIELDS + or type(plan.get("schema_version")) is not int + or plan["schema_version"] != 1 + or plan.get("operation_kind") != "maintenance" + ): + raise CloudError("Central maintenance plan schema is invalid") + plan_id = plan.get("operation_id") + if not isinstance(plan_id, str) or not re.fullmatch(r"[0-9a-f]{32}", plan_id): + raise CloudError("Central maintenance plan identity is invalid") + if path.name != f"{plan_id}.json" or path.parent.name != "retention-plans": + raise CloudError("Central maintenance plan path does not match its identity") + if plan.get("repository_lease_id") != self.repository_lease_id: + raise CloudError("Central maintenance plan belongs to a different repository lease") + if not isinstance(plan.get("target"), dict) or set(plan["target"]) != { + "endpoint_url", "bucket", "prefix", "repository_id" + }: + raise CloudError("Central maintenance plan target is invalid") + try: + _repository_identity(plan["target"]["repository_id"]) + except CloudError: + raise CloudError("Central maintenance plan target is invalid") from None + expected_target = { + "endpoint_url": self.adapter.endpoint_url, + "bucket": self.adapter.bucket, + "prefix": self.adapter.prefix, + "repository_id": plan.get("target", {}).get("repository_id") + if isinstance(plan.get("target"), dict) + else None, + } + if plan.get("target") != expected_target: + raise CloudError("Central maintenance plan target is invalid") + for name in ( + "policy_sha256", + "grant_sha256", + "catalog_sha256", + "protected_sha256", + "inventory_sha256", + "resource_inventory_sha256", + "snapshot_inventory_sha256", + "bindings_sha256", + "credential_identity_sha256", + ): + if not isinstance(plan.get(name), str) or not _FULL_ID.fullmatch(plan[name]): + raise CloudError("Central maintenance plan digest is invalid") + for name in ("created_at", "deadline", "grant_expires_at"): + _parse_time(plan.get(name)) + for name in ( + "accepted_manifest_digests", + "selected_manifest_digests", + ): + values = plan.get(name) + if ( + not isinstance(values, list) + or any(not isinstance(value, str) or not _FULL_ID.fullmatch(value) for value in values) + or values != sorted(set(values)) + ): + raise CloudError("Central maintenance recovery selection is invalid") + for name in ("accepted_snapshot_ids", "protected_snapshot_ids", "visible_snapshot_ids", "target_snapshot_ids"): + _maintenance_snapshot_ids(plan.get(name), label=name.replace("_", " ")) + accepted = set(plan["accepted_snapshot_ids"]) + protected = set(plan["protected_snapshot_ids"]) + visible = set(plan["visible_snapshot_ids"]) + targets = set(plan["target_snapshot_ids"]) + if ( + not protected <= accepted + or not protected <= visible + or not targets <= (accepted & visible) + or targets & protected + or targets != (accepted & visible) - protected + or plan["config_repository_id"] != plan["target"]["repository_id"] + or plan["snapshot_inventory_sha256"] + != _maintenance_snapshot_digest(plan["config_repository_id"], plan["visible_snapshot_ids"]) + ): + raise CloudError("Central maintenance snapshot selection is invalid") + limits = plan.get("apply_limits") + if not isinstance(limits, dict) or set(limits) != {"max_apply_objects", "max_apply_bytes"} or any( + type(value) is not int or value < 1 for value in limits.values() + ): + raise CloudError("Central maintenance apply bounds are invalid") + resources = plan.get("resources") + if not isinstance(resources, dict) or set(resources) != { + "inventory_version_count", + "current_payload_count", + "current_payload_bytes", + "max_apply_objects", + "max_apply_bytes", + "resource_budget", + "timeout_seconds", + }: + raise CloudError("Central maintenance resource evidence is invalid") + for name in ( + "inventory_version_count", + "current_payload_count", + "current_payload_bytes", + "max_apply_objects", + "max_apply_bytes", + ): + if type(resources[name]) is not int or resources[name] < 0: + raise CloudError("Central maintenance resource evidence is invalid") + if ( + resources["current_payload_count"] > limits["max_apply_objects"] + or resources["current_payload_bytes"] > limits["max_apply_bytes"] + or resources["max_apply_objects"] != limits["max_apply_objects"] + or resources["max_apply_bytes"] != limits["max_apply_bytes"] + ): + raise CloudError("Central maintenance resource bounds are invalid") + if ( + not isinstance(plan.get("maintenance"), dict) + or set(plan["maintenance"]) != {"mode", "resource_budget", "timeout_seconds"} + or plan["maintenance"]["mode"] != _MAINTENANCE_MODE + or plan["maintenance"]["resource_budget"] != resources["resource_budget"] + or type(plan["maintenance"]["timeout_seconds"]) is not int + or plan["maintenance"]["timeout_seconds"] != resources["timeout_seconds"] + or not 1 <= resources["timeout_seconds"] <= 86400 + ): + raise CloudError("Central maintenance mode or budget is invalid") + if not isinstance(resources["resource_budget"], dict) or set(resources["resource_budget"]) != { + "schema_version", "staging_max_bytes", "minimum_free_bytes", "cache_max_bytes" + }: + raise CloudError("Central maintenance resource budget is invalid") + try: + from .models import ResourceBudget + + budget = ResourceBudget.parse(resources["resource_budget"]) + except Exception: + raise CloudError("Central maintenance resource budget is invalid") from None + if { + "schema_version": 1, + "staging_max_bytes": budget.staging_max_bytes, + "minimum_free_bytes": budget.minimum_free_bytes, + "cache_max_bytes": budget.cache_max_bytes, + } != resources["resource_budget"]: + raise CloudError("Central maintenance resource budget is invalid") + if type(plan.get("protected_version_count")) is not int or plan["protected_version_count"] < 0: + raise CloudError("Central maintenance protected-version count is invalid") + if ( + type(plan.get("version_count")) is not int + or plan["version_count"] < 0 + or plan["version_count"] != resources["inventory_version_count"] + ): + raise CloudError("Central maintenance inventory count is invalid") + return plan, raw, _retention_digest(raw) + + def _assert_plan_current(self, plan: Mapping[str, Any], *, require_live: bool) -> None: + policy, grant, state_dir, now, policy_sha256, grant_sha256 = self._authority( + allow_expired=not require_live + )[:6] + if ( + policy_sha256 != plan["policy_sha256"] + or grant_sha256 != plan["grant_sha256"] + or policy["policy_id"] != plan["policy_id"] + or policy["revision"] != plan["policy_revision"] + or grant.get("action") != "maintain_current_repository" + or _retention_target(policy) != plan["target"] + or state_dir != self._selection_data["state_dir"] + ): + raise CloudError("Central maintenance authority changed after planning") + catalog, raw = self.owner._load_catalog(state_dir, policy) + del catalog + if _retention_digest(raw) != plan["catalog_sha256"]: + raise CloudError("Central maintenance catalog changed after planning") + if now >= _parse_time(plan["deadline"]) or now >= _parse_time(plan["grant_expires_at"]): + if require_live: + raise CloudError("Central maintenance plan or grant has expired") + + def plan( + self, + restic_evidence: Mapping[str, Any], + *, + bindings_sha256: str, + credential_identity: Mapping[str, Any], + ) -> dict[str, Any]: + """Create an immutable plan for accepted, visible, unprotected snapshots.""" + if self._selection_data is None or self.selection is None: + raise CloudError("Central maintenance session is not active") + captured = self._capture() + self._compare_capture(self._selection_data, captured) + evidence = _maintenance_restic_evidence( + restic_evidence, + expected_repository_id=captured["policy"]["repository_id"], + ) + if not isinstance(bindings_sha256, str) or not _FULL_ID.fullmatch(bindings_sha256): + raise CloudError("Central maintenance private bindings digest is invalid") + credential_digest = _maintenance_credential_digest(credential_identity) + visible = set(evidence["visible_snapshot_ids"]) + protected = set(captured["protected_snapshot_ids"]) + if not protected <= visible: + raise CloudError("A protected recovery snapshot is missing from the Restic repository") + self._assert_no_unresolved_maintenance(captured["state_dir"]) + targets = sorted( + visible.intersection(captured["accepted_snapshot_ids"]) - protected + ) + policy = captured["policy"] + grant = captured["grant"] + created = datetime.now(timezone.utc) + grant_expiry = _parse_time(grant["expires_at"]) + deadline = min( + created + timedelta(seconds=policy["plan_max_age_seconds"]), + grant_expiry, + ) + operation_id = uuid.uuid4().hex + plan = { + "schema_version": 1, + "operation_kind": "maintenance", + "operation_id": operation_id, + "created_at": _iso(created), + "deadline": _iso(deadline), + "target": _retention_target(policy), + "repository_lease_id": self.repository_lease_id, + "policy_id": policy["policy_id"], + "policy_revision": policy["revision"], + "policy_sha256": captured["policy_sha256"], + "grant_sha256": captured["grant_sha256"], + "grant_expires_at": _iso(grant_expiry), + "catalog_sha256": captured["catalog_sha256"], + "accepted_manifest_digests": captured["accepted_manifest_digests"], + "selected_manifest_digests": captured["selected_manifest_digests"], + "accepted_snapshot_ids": captured["accepted_snapshot_ids"], + "protected_snapshot_ids": captured["protected_snapshot_ids"], + "protected_sha256": captured["protected_sha256"], + "protected_version_count": captured["protected_version_count"], + "inventory_sha256": captured["inventory_sha256"], + "version_count": captured["version_count"], + "resource_inventory_sha256": captured["resource_inventory_sha256"], + "resources": captured["resources"], + "apply_limits": { + "max_apply_objects": captured["resources"]["max_apply_objects"], + "max_apply_bytes": captured["resources"]["max_apply_bytes"], + }, + "maintenance": policy["maintenance"], + "config_repository_id": evidence["config_repository_id"], + "visible_snapshot_ids": evidence["visible_snapshot_ids"], + "snapshot_inventory_sha256": evidence["snapshot_inventory_sha256"], + "target_snapshot_ids": targets, + "bindings_sha256": bindings_sha256, + "credential_identity_sha256": credential_digest, + } + raw = _write_immutable_plan( + self._plans_directory(captured["state_dir"], create=True) + / f"{operation_id}.json", + plan, + ) + return { + "status": "planned", + "plan_id": operation_id, + "plan_sha256": _retention_digest(raw), + "accepted_snapshot_ids": list(captured["accepted_snapshot_ids"]), + "protected_snapshot_ids": list(captured["protected_snapshot_ids"]), + "target_snapshot_ids": targets, + "protected_version_sha256": captured["protected_sha256"], + "resources": captured["resources"], + "deadline": plan["deadline"], + } + + def _read_journal(self, plan_id: str, plan_sha256: str) -> tuple[dict[str, Any], Path]: + path = self._journal_path(plan_id) + journal, _raw = _read_private_json(path) + if ( + set(journal) != _MAINTENANCE_JOURNAL_FIELDS + or type(journal.get("schema_version")) is not int + or journal["schema_version"] != 1 + or journal.get("operation_kind") != "maintenance" + or journal.get("operation_id") != plan_id + or journal.get("plan_sha256") != plan_sha256 + or not isinstance(journal.get("status"), str) + ): + raise CloudError("Central maintenance operation journal is invalid") + _parse_time(journal.get("updated_at")) + phases = journal.get("phases") + if not isinstance(phases, dict) or set(phases) != set(_MAINTENANCE_PHASES): + raise CloudError("Central maintenance phase journal is invalid") + for phase in _MAINTENANCE_PHASES: + value = phases[phase] + if value is not None and ( + not isinstance(value, dict) + or set(value) != {"status", "evidence"} + or value.get("status") not in {"intent", "completed", "failed", "uncertain"} + or (value["evidence"] is not None and not isinstance(value["evidence"], dict)) + ): + raise CloudError("Central maintenance phase journal is invalid") + if journal.get("reconciliation") is not None and not isinstance( + journal["reconciliation"], dict + ): + raise CloudError("Central maintenance reconciliation record is invalid") + return journal, path + + @staticmethod + def _journal_value(plan: Mapping[str, Any], plan_sha256: str) -> dict[str, Any]: + return { + "schema_version": 1, + "operation_kind": "maintenance", + "operation_id": plan["operation_id"], + "plan_sha256": plan_sha256, + "status": "prepared", + "updated_at": _iso(datetime.now(timezone.utc)), + "phases": {phase: None for phase in _MAINTENANCE_PHASES}, + "reconciliation": None, + } + + @staticmethod + def _save_maintenance_journal(path: Path, journal: dict[str, Any]) -> None: + journal["updated_at"] = _iso(datetime.now(timezone.utc)) + _write_private_json(path, journal, replace=os.path.lexists(path)) + + def _private_plan( + self, plan_id: str, expected_sha256: str + ) -> tuple[dict[str, Any], str]: + if not isinstance(expected_sha256, str) or not _FULL_ID.fullmatch(expected_sha256): + raise CloudError("Central maintenance expected plan digest is invalid") + plan, _raw, digest = self._read_plan(self._plan_path(plan_id)) + if digest != expected_sha256: + raise CloudError("Central maintenance plan digest differs from the reviewed plan") + return plan, digest + + def reconciliation_plan( + self, + plan_id: str, + expected_sha256: str, + *, + bindings_sha256: str, + credential_identity: Mapping[str, Any], + ) -> dict[str, Any]: + """Read an exact expired attempt for observation, never mutation replay.""" + if not self.reconcile_only: + raise CloudError("Central maintenance reconciliation requires read-only mode") + plan, _digest = self._private_plan(plan_id, expected_sha256) + self._assert_plan_current(plan, require_live=False) + if (plan["repository_lease_id"] != self.repository_lease_id + or plan["bindings_sha256"] != bindings_sha256 + or plan["credential_identity_sha256"] + != _maintenance_credential_digest(credential_identity)): + raise CloudError("Central maintenance reconciliation identity changed") + return plan + + def validate_plan( + self, + plan_id: str, + expected_sha256: str, + restic_evidence: Mapping[str, Any], + *, + bindings_sha256: str, + credential_identity: Mapping[str, Any], + ) -> dict[str, Any]: + """Revalidate immutable authority, catalog, S3 inventory, and Restic inputs.""" + plan, plan_sha256 = self._private_plan(plan_id, expected_sha256) + self._assert_plan_current(plan, require_live=True) + evidence = _maintenance_restic_evidence( + restic_evidence, + expected_repository_id=plan["config_repository_id"], + ) + if evidence != { + "config_repository_id": plan["config_repository_id"], + "visible_snapshot_ids": plan["visible_snapshot_ids"], + "snapshot_inventory_sha256": plan["snapshot_inventory_sha256"], + }: + raise CloudError("Central maintenance Restic inventory changed after planning") + if not isinstance(bindings_sha256, str) or bindings_sha256 != plan["bindings_sha256"]: + raise CloudError("Central maintenance private bindings changed after planning") + if _maintenance_credential_digest(credential_identity) != plan["credential_identity_sha256"]: + raise CloudError("Central maintenance credential identity changed after planning") + captured = self._capture() + self._compare_capture(self._selection_data, captured) + comparisons = { + "policy_sha256": captured["policy_sha256"], + "grant_sha256": captured["grant_sha256"], + "catalog_sha256": captured["catalog_sha256"], + "accepted_manifest_digests": captured["accepted_manifest_digests"], + "selected_manifest_digests": captured["selected_manifest_digests"], + "accepted_snapshot_ids": captured["accepted_snapshot_ids"], + "protected_snapshot_ids": captured["protected_snapshot_ids"], + "protected_sha256": captured["protected_sha256"], + "protected_version_count": captured["protected_version_count"], + "inventory_sha256": captured["inventory_sha256"], + "version_count": captured["version_count"], + "resource_inventory_sha256": captured["resource_inventory_sha256"], + "resources": captured["resources"], + } + if any(plan[name] != value for name, value in comparisons.items()): + raise CloudError("Central maintenance inputs drifted after planning") + journal_path = self._journal_path(plan_id) + if os.path.lexists(journal_path): + journal, _path = self._read_journal(plan_id, plan_sha256) + if any( + journal["phases"][phase] is not None + for phase in _MAINTENANCE_MUTATION_PHASES + ): + raise CloudError("Central maintenance mutation already started; plan cannot be replayed") + if journal["status"] not in {"prepared", "active"}: + raise CloudError("Central maintenance plan has an unresolved attempt") + return plan + + @staticmethod + def _expected_after_ids(plan: Mapping[str, Any]) -> list[str]: + return sorted(set(plan["visible_snapshot_ids"]) - set(plan["target_snapshot_ids"])) + + @staticmethod + def _health_observation( + value: Any, + plan: Mapping[str, Any], + *, + label: str, + ) -> dict[str, Any]: + if not isinstance(value, Mapping) or set(value) != { + "repository_health", + "config_repository_id", + "visible_snapshot_ids", + "snapshot_inventory_sha256", + }: + raise CloudError(f"Central maintenance {label} observation schema is invalid") + if value.get("repository_health") != "healthy": + raise CloudError(f"Central maintenance {label} repository health is not proven") + evidence = _maintenance_restic_evidence( + { + "config_repository_id": value.get("config_repository_id"), + "visible_snapshot_ids": value.get("visible_snapshot_ids"), + "snapshot_inventory_sha256": value.get("snapshot_inventory_sha256"), + }, + expected_repository_id=plan["config_repository_id"], + ) + return {"repository_health": "healthy", **evidence} + + def _phase_evidence( + self, + plan: Mapping[str, Any], + phase: str, + evidence: Any, + ) -> dict[str, Any]: + if phase == "check_before": + checked = self._health_observation(evidence, plan, label="pre-maintenance") + expected = { + "config_repository_id": plan["config_repository_id"], + "visible_snapshot_ids": plan["visible_snapshot_ids"], + "snapshot_inventory_sha256": plan["snapshot_inventory_sha256"], + } + if any(checked[name] != value for name, value in expected.items()): + raise CloudError("Central maintenance pre-check inventory differs from the plan") + return checked + if phase == "forget": + if not isinstance(evidence, Mapping) or set(evidence) != {"forgotten_snapshot_ids"}: + raise CloudError("Central maintenance forget evidence schema is invalid") + forgotten = _maintenance_snapshot_ids( + evidence["forgotten_snapshot_ids"], label="forgotten snapshot IDs" + ) + if forgotten != plan["target_snapshot_ids"]: + raise CloudError("Central maintenance forget evidence differs from authorized targets") + return {"forgotten_snapshot_ids": forgotten} + if phase == "verify_snapshots": + checked = _maintenance_restic_evidence( + evidence, + expected_repository_id=plan["config_repository_id"], + ) + if checked["visible_snapshot_ids"] != self._expected_after_ids(plan): + raise CloudError("Central maintenance snapshot verification found unexpected snapshots") + return checked + if phase == "prune": + if not isinstance(evidence, Mapping) or set(evidence) != { + "result", "evidence_sha256" + } or evidence.get("result") != "completed" or not isinstance( + evidence.get("evidence_sha256"), str + ) or not _FULL_ID.fullmatch(evidence["evidence_sha256"]): + raise CloudError("Central maintenance prune evidence is invalid") + return { + "result": "completed", + "evidence_sha256": evidence["evidence_sha256"], + } + if phase == "check_after": + checked = self._health_observation(evidence, plan, label="post-maintenance") + if checked["visible_snapshot_ids"] != self._expected_after_ids(plan): + raise CloudError("Central maintenance post-check snapshot inventory is unexpected") + return checked + if phase == "protected_versions": + if not isinstance(evidence, Mapping) or set(evidence) != { + "protected_version_sha256", "protected_version_count" + }: + raise CloudError("Central maintenance protected-version evidence is invalid") + if ( + evidence.get("protected_version_sha256") != plan["protected_sha256"] + or type(evidence.get("protected_version_count")) is not int + or evidence["protected_version_count"] != plan["protected_version_count"] + or self._last_verified_protected != dict(evidence) + ): + raise CloudError("Central maintenance protected-version evidence was not freshly verified") + return dict(evidence) + raise CloudError("Unknown central maintenance phase") + + @staticmethod + def _mutation_started(journal: Mapping[str, Any]) -> bool: + return any(journal["phases"][phase] is not None for phase in _MAINTENANCE_MUTATION_PHASES) + + def record_phase( + self, + plan: Mapping[str, Any], + phase: str, + status: str, + *, + evidence: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + """Durably record a fixed maintenance phase without executing it.""" + if phase not in _MAINTENANCE_PHASES or status not in { + "intent", "completed", "failed", "uncertain" + }: + raise CloudError("Central maintenance phase or status is invalid") + if not isinstance(plan, Mapping) or set(plan) != _MAINTENANCE_PLAN_FIELDS: + raise CloudError("Central maintenance private plan is invalid") + plan_id = plan.get("operation_id") + plan_digest = _retention_digest(_retention_json_bytes(plan)) + stored, stored_digest = self._private_plan(plan_id, plan_digest) + if stored != dict(plan) or stored_digest != plan_digest: + raise CloudError("Central maintenance private plan differs from durable plan") + self._assert_plan_current(stored, require_live=(status == "intent")) + journal_path = self._journal_path(plan_id) + if os.path.lexists(journal_path): + journal, journal_path = self._read_journal(plan_id, plan_digest) + else: + journal = None + phases = _MAINTENANCE_PHASES + index = phases.index(phase) + if status == "intent": + if evidence is not None: + raise CloudError("Central maintenance phase intent cannot contain result evidence") + if journal is None: + if index != 0: + raise CloudError("Central maintenance phases must start with check_before") + journal = self._journal_value(stored, plan_digest) + if journal["status"] not in {"prepared", "active"}: + raise CloudError("Central maintenance operation has an unresolved phase") + if journal["phases"][phase] is not None: + raise CloudError("Central maintenance phase intent was already recorded") + if any(journal["phases"][prior] is None or journal["phases"][prior]["status"] != "completed" + for prior in phases[:index]): + raise CloudError("Central maintenance phase order is invalid") + journal["phases"][phase] = {"status": "intent", "evidence": None} + journal["status"] = "active" + else: + if journal is None: + raise CloudError("Central maintenance phase completion has no durable intent") + if journal["status"] not in {"prepared", "active"}: + raise CloudError("Central maintenance operation has an unresolved phase") + current = journal["phases"][phase] + if current is None or current["status"] != "intent": + raise CloudError("Central maintenance phase has no matching durable intent") + if evidence is not None and not isinstance(evidence, Mapping): + raise CloudError("Central maintenance phase evidence must be an object") + if status == "completed": + clean_evidence = self._phase_evidence(stored, phase, evidence) + journal["phases"][phase] = {"status": "completed", "evidence": clean_evidence} + journal["status"] = "active" + else: + if evidence is not None: + raise CloudError("Failed maintenance phases cannot store raw result evidence") + journal["phases"][phase] = {"status": status, "evidence": None} + journal["status"] = "uncertain" if status == "uncertain" else "failed" + self._save_maintenance_journal(journal_path, journal) + return { + "status": journal["status"], + "plan_id": plan_id, + "phase": phase, + "phase_status": journal["phases"][phase]["status"], + } + + def verify_protected_versions( + self, *, plan: Mapping[str, Any] | None = None + ) -> dict[str, Any]: + """Reinventory and prove the exact selected recovery versions are present.""" + if self._selection_data is None: + raise CloudError("Central maintenance session is not active") + if plan is None: + captured = self._capture(allow_expired=True) + for name in ( + "policy_sha256", "grant_sha256", "catalog_sha256", + "accepted_manifest_digests", "selected_manifest_digests", + "accepted_snapshot_ids", "protected_snapshot_ids", + "protected_sha256", "protected_version_count", + ): + if captured[name] != self._selection_data[name]: + raise CloudError("Central maintenance protected recovery set changed") + digest, count = captured["protected_sha256"], captured["protected_version_count"] + else: + self._assert_plan_current(plan, require_live=False) + if (plan["accepted_manifest_digests"] != self._selection_data["accepted_manifest_digests"] + or not set(plan["selected_manifest_digests"]) <= set(plan["accepted_manifest_digests"])): + raise CloudError("Central maintenance accepted recovery set changed") + policy, _grant, state_dir, now = self._authority(allow_expired=True)[:4] + with tempfile.TemporaryDirectory(prefix="maintenance-protected-", dir=state_dir) as name: + temporary = Path(name) + os.chmod(temporary, 0o700) + records = self.owner._pin_records(self.record_dirs, policy, temporary, now) + if sorted(record["manifest_sha256"] for record in records) != plan["accepted_manifest_digests"]: + raise CloudError("Central maintenance accepted records changed") + inventory, _digest, _count = self.owner._inventory(temporary / "inventory.sqlite3") + try: + protected, digest, count = self.owner._protected_versions( + records, set(plan["selected_manifest_digests"]), inventory, + temporary / "protected.sqlite3", + ) + protected.close() + finally: + inventory.close() + if digest != plan["protected_sha256"] or count != plan["protected_version_count"]: + raise CloudError("Central maintenance planned protected versions changed") + result = {"protected_version_sha256": digest, "protected_version_count": count} + self._last_verified_protected = result + return result + + def finish_reconciliation( + self, + plan: Mapping[str, Any], + observations: Mapping[str, Any], + ) -> dict[str, Any]: + """Reconcile read-only observations; never infer prune completion from health alone.""" + if not isinstance(plan, Mapping) or set(plan) != _MAINTENANCE_PLAN_FIELDS: + raise CloudError("Central maintenance private plan is invalid") + plan_id = plan.get("operation_id") + plan_digest = _retention_digest(_retention_json_bytes(plan)) + stored, stored_digest = self._private_plan(plan_id, plan_digest) + if stored != dict(plan) or stored_digest != plan_digest: + raise CloudError("Central maintenance private plan differs from durable plan") + self._assert_plan_current(stored, require_live=False) + if not isinstance(observations, Mapping) or set(observations) != { + "schema_version", + "repository_health", + "config_repository_id", + "visible_snapshot_ids", + "snapshot_inventory_sha256", + } or type(observations.get("schema_version")) is not int or observations["schema_version"] != 1: + raise CloudError("Central maintenance reconciliation observation schema is invalid") + checked = self._health_observation( + {name: observations[name] for name in ( + "repository_health", "config_repository_id", "visible_snapshot_ids", "snapshot_inventory_sha256" + )}, + stored, + label="reconciliation", + ) + visible = set(checked["visible_snapshot_ids"]) + initial = set(stored["visible_snapshot_ids"]) + targets = set(stored["target_snapshot_ids"]) + expected_survivors = initial - targets + if not expected_survivors <= visible or visible - initial: + raise CloudError("Central maintenance reconciliation found repository snapshot drift") + protected = set(stored["protected_snapshot_ids"]) + if not protected <= visible: + raise CloudError("Central maintenance reconciliation cannot prove protected snapshots") + protected_evidence = self.verify_protected_versions(plan=stored) + journal, journal_path = self._read_journal(plan_id, plan_digest) + completed = all( + isinstance(journal["phases"][phase], dict) + and journal["phases"][phase]["status"] == "completed" + for phase in _MAINTENANCE_PHASES + ) + if completed and visible != expected_survivors: + raise CloudError("Completed central maintenance still has authorized target snapshots") + if completed and journal["phases"]["protected_versions"]["evidence"] != protected_evidence: + raise CloudError("Completed central maintenance protected-version evidence changed") + if completed: + status = "complete" + outcome = "succeeded" + elif self._mutation_started(journal): + status = "partial_abandoned" + outcome = "abandoned" + else: + status = "abandoned" + outcome = "abandoned" + journal["status"] = status + journal["reconciliation"] = { + "outcome": outcome, + "repository_health": "healthy", + "snapshot_inventory_sha256": checked["snapshot_inventory_sha256"], + "visible_snapshot_ids": checked["visible_snapshot_ids"], + **protected_evidence, + } + self._save_maintenance_journal(journal_path, journal) + return { + "status": status, + "outcome": outcome, + "plan_id": plan_id, + "target_snapshot_ids": stored["target_snapshot_ids"], + "remaining_target_snapshot_ids": sorted(targets & visible), + "protected_version_sha256": protected_evidence["protected_version_sha256"], + "protected_version_count": protected_evidence["protected_version_count"], + } + __all__ = [ "CheckpointRecord", "CloudError", + "CloudMaintenanceSelection", "LifecycleInspection", "RestoreRecord", "S3CloudAdapter", diff --git a/bbackup/data/cli-skills.md b/bbackup/data/cli-skills.md index af91b0c..04b74f6 100644 --- a/bbackup/data/cli-skills.md +++ b/bbackup/data/cli-skills.md @@ -1,6 +1,6 @@ # CLI skills catalog -> Generated from the bbackup/bbman CLI metadata. Version: 2.0.3. This catalog is authoritative for this version. +> Generated from the bbackup/bbman CLI metadata. Version: 2.0.4. This catalog is authoritative for this version. > Source: `bbackup/cli_metadata.py` rendered by `scripts/generate_cli_skills.py`; run the generator's `--check` mode before publishing. ## bbackup diff --git a/bbackup/data/production-schemas.json b/bbackup/data/production-schemas.json index 1f11a58..da63daf 100644 --- a/bbackup/data/production-schemas.json +++ b/bbackup/data/production-schemas.json @@ -1,4 +1,315 @@ { + "cloud-maintenance apply": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "grant_file": { + "minLength": 1, + "type": "string" + }, + "maintenance_bindings_file": { + "minLength": 1, + "type": "string" + }, + "plan_id": { + "minLength": 1, + "type": "string" + }, + "plan_sha256": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "quiescent_window_confirmed": { + "default": false, + "type": "boolean" + }, + "records_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "grant_file", + "records_file", + "maintenance_bindings_file", + "plan_id", + "plan_sha256" + ], + "type": "object" + }, + "cloud-maintenance catalog-enroll": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "grant_file": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "grant_file" + ], + "type": "object" + }, + "cloud-maintenance plan": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "grant_file": { + "minLength": 1, + "type": "string" + }, + "maintenance_bindings_file": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "quiescent_window_confirmed": { + "default": false, + "type": "boolean" + }, + "records_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "grant_file", + "records_file", + "maintenance_bindings_file" + ], + "type": "object" + }, + "cloud-maintenance reconcile": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "grant_file": { + "minLength": 1, + "type": "string" + }, + "maintenance_bindings_file": { + "minLength": 1, + "type": "string" + }, + "plan_id": { + "minLength": 1, + "type": "string" + }, + "plan_sha256": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "quiescent_window_confirmed": { + "default": false, + "type": "boolean" + }, + "records_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "grant_file", + "records_file", + "maintenance_bindings_file", + "plan_id", + "plan_sha256" + ], + "type": "object" + }, + "cloud-maintenance records-enroll": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "grant_file": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "records_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "grant_file", + "records_file" + ], + "type": "object" + }, + "cloud-retention apply": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "grant_file": { + "minLength": 1, + "type": "string" + }, + "plan_id": { + "minLength": 1, + "type": "string" + }, + "plan_sha256": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "quiescent_window_confirmed": { + "default": false, + "type": "boolean" + }, + "records_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "grant_file", + "records_file", + "plan_id", + "plan_sha256" + ], + "type": "object" + }, + "cloud-retention catalog-enroll": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "grant_file": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "grant_file" + ], + "type": "object" + }, + "cloud-retention plan": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "grant_file": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "quiescent_window_confirmed": { + "default": false, + "type": "boolean" + }, + "records_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "grant_file", + "records_file" + ], + "type": "object" + }, + "cloud-retention reconcile": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "plan_file": { + "minLength": 1, + "type": "string" + }, + "policy_file": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "policy_file", + "plan_file" + ], + "type": "object" + }, "config migrate": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -32,6 +343,92 @@ "required": [], "type": "object" }, + "coverage check": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "name": { + "minLength": 1, + "type": "string" + }, + "quiet_administrative_window_confirmed": { + "default": false, + "type": "boolean" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "coverage discover": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "name": { + "minLength": 1, + "type": "string" + }, + "quiet_administrative_window_confirmed": { + "default": false, + "type": "boolean" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "coverage recovery-evidence": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "evidence_file": { + "minLength": 1, + "type": "string" + }, + "name": { + "minLength": 1, + "type": "string" + }, + "quiet_administrative_window_confirmed": { + "default": false, + "type": "boolean" + }, + "snapshot": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "name", + "snapshot" + ], + "type": "object" + }, + "coverage review": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "candidate_digest": { + "minLength": 1, + "type": "string" + }, + "name": { + "minLength": 1, + "type": "string" + }, + "quiet_administrative_window_confirmed": { + "default": false, + "type": "boolean" + } + }, + "required": [ + "name", + "candidate_digest" + ], + "type": "object" + }, "databases reconcile": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -222,6 +619,10 @@ "name": { "minLength": 1, "type": "string" + }, + "quiet_administrative_window_confirmed": { + "default": false, + "type": "boolean" } }, "required": [ @@ -229,6 +630,68 @@ ], "type": "object" }, + "recovery accept": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "bucket": { + "minLength": 1, + "type": "string" + }, + "endpoint": { + "minLength": 1, + "type": "string" + }, + "kit": { + "minLength": 1, + "type": "string" + }, + "prefix": { + "minLength": 1, + "type": "string" + }, + "region": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + }, + "repository_id": { + "minLength": 1, + "type": "string" + }, + "snapshot": { + "minLength": 1, + "type": "string" + }, + "source_signer": { + "minLength": 1, + "type": "string" + }, + "target": { + "minLength": 1, + "type": "string" + }, + "verifier": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "kit", + "target", + "source_signer", + "verifier", + "repository", + "repository_id", + "snapshot", + "bucket", + "endpoint" + ], + "type": "object" + }, "recovery b2-inspect": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -529,6 +992,30 @@ ], "type": "object" }, + "runs discard-exports": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "artifact_record_sha256": { + "minLength": 1, + "type": "string" + }, + "operation_id": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository", + "operation_id", + "artifact_record_sha256" + ], + "type": "object" + }, "runs list": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -558,6 +1045,10 @@ "minLength": 1, "type": "string" }, + "quiescent_window_confirmed": { + "default": false, + "type": "boolean" + }, "repository": { "minLength": 1, "type": "string" @@ -574,6 +1065,29 @@ ], "type": "object" }, + "runs retained-exports": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "limit": { + "maximum": 1000, + "minimum": 1, + "type": "integer" + }, + "offset": { + "minimum": 0, + "type": "integer" + }, + "repository": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "repository" + ], + "type": "object" + }, "runs stream": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -640,6 +1154,25 @@ ], "type": "object" }, + "service render-system": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "executable": { + "minLength": 1, + "type": "string" + }, + "name": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "name", + "executable" + ], + "type": "object" + }, "setup": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, diff --git a/bbackup/enrollment.py b/bbackup/enrollment.py index 5f95d24..3f8f32e 100644 --- a/bbackup/enrollment.py +++ b/bbackup/enrollment.py @@ -69,6 +69,19 @@ def evidence_bytes(policy: bytes, bindings: bytes) -> dict[str, object]: "bindings_digest": canonical_digest(bindings)} +def coverage_evidence_bytes(policy: bytes, bindings: bytes, *, declaration_digest: str, + scope_policy_digest: str, baseline_digest: str) -> dict[str, object]: + """Bind a separately reviewed coverage baseline to exact host enrollment. + + Ordinary ``write_bytes`` never writes or approves this coverage receipt. + """ + if any(not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in (declaration_digest, scope_policy_digest, baseline_digest)): + raise ConfigurationError("Invalid coverage evidence digest") + return {**evidence_bytes(policy, bindings), "declaration_digest": declaration_digest, + "scope_policy_digest": scope_policy_digest, "baseline_digest": baseline_digest} + + def evidence(config_path: Path, bindings_path: Path) -> dict[str, object]: return evidence_bytes(*read_pair(config_path, bindings_path)) diff --git a/bbackup/linux_coverage.py b/bbackup/linux_coverage.py new file mode 100644 index 0000000..825b18e --- /dev/null +++ b/bbackup/linux_coverage.py @@ -0,0 +1,1187 @@ +"""Machine-bound Linux coverage inventory and exact reviewed scope. + +Private declarations and candidates stay beneath the existing state root. This +module inventories names and metadata and runs integrity checks for reviewed +SQLite roots; it never exposes protected file contents. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +import base64 +import hashlib +import json +import os +from pathlib import Path +import re +import sqlite3 +import stat +import tempfile +from typing import Any + +from .enrollment import (coverage_evidence_bytes, evidence_bytes, policy_lease, + verify_bytes) +from .models import ConfigurationError, strict_json +from .postgresql_coverage import (PostgreSQLCoverageContext, + inspect_postgresql_coverage) + + +_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z") +_ACCOUNT = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,63}\$?\Z") +_SYSTEMD_UNIT_PART = re.compile(r"(?:[A-Za-z0-9:_.-]|\\x[0-9A-Fa-f]{2})+\Z") +_DIGEST = re.compile(r"[0-9a-f]{64}\Z") +_METADATA = {"numeric_owner", "mode", "acl", "xattr", "links"} +_MAX_INVENTORY = 16 * 1024 * 1024 +_MAX_SYSTEMD_UNIT_NAME = 255 +_MAX_ENTRIES = 100_000 +_MAX_SQLITE_XATTRS = 256 +_MAX_SQLITE_XATTR_BYTES = 1024 * 1024 + + +class CoverageError(RuntimeError): + """Sanitized coverage failure; private paths never appear in messages.""" + + +class _SQLiteCoverageError(CoverageError): + def __init__(self, reason: str): + self.reason = reason + super().__init__(reason) + + +@dataclass(frozen=True) +class InventoryPaths: + passwd: Path = Path("/etc/passwd") + group: Path = Path("/etc/group") + mountinfo: Path = Path("/proc/self/mountinfo") + packages: Path = Path("/var/lib/dpkg/status") + application_units: Path = Path("/etc/systemd/system") + + +@dataclass(frozen=True) +class CoverageCheck: + coverage_status: str + reasons: tuple[str, ...] + baseline_digest: str | None + candidate_digest: str | None + counts: dict[str, int] + approved_roots: tuple[tuple[str, Path, str, str], ...] = field(default=(), repr=False) + approved_exclusions: tuple[Path, ...] = field(default=(), repr=False) + approved_sqlite: tuple[dict[str, Any], ...] = field(default=(), repr=False) + sqlite_capture_exclusions: tuple[Path, ...] = field(default=(), repr=False) + recovery_status: str = "untested" + approved_postgresql: tuple[dict[str, Any], ...] = field(default=(), repr=False) + postgresql_capture_exclusions: tuple[Path, ...] = field(default=(), repr=False) + + @property + def capture_ready(self) -> bool: + return self.coverage_status == "complete" and bool( + self.approved_roots or self.approved_postgresql) + + def public_result(self) -> dict[str, object]: + return {"coverage_status": self.coverage_status, "recovery_status": self.recovery_status, + "capture_ready": self.capture_ready, "reasons": list(self.reasons), + "baseline_digest": self.baseline_digest, "candidate_digest": self.candidate_digest, + "counts": self.counts, + "next_action": ("Capture only the reviewed source mappings" if self.capture_ready + else "Review current coverage discovery and exact exclusions")} + + +def _canonical(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + + +def _digest(value: Any) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _identifier(value: Any, description: str) -> str: + if not isinstance(value, str) or not _NAME.fullmatch(value): + raise CoverageError(f"Invalid {description} identifier") + return value + + +def _systemd_service_stem(value: Any) -> str: + """Validate a service unit name without its required .service suffix.""" + if (not isinstance(value, str) or not value + or len(value) + len(".service") > _MAX_SYSTEMD_UNIT_NAME + or "/" in value + or any(ord(char) < 0x20 or ord(char) == 0x7f for char in value) + or r"\x00" in value + or value.count("@") > 1): + raise CoverageError("Invalid application unit identifier") + prefix, has_instance, instance = value.partition("@") + if (not _SYSTEMD_UNIT_PART.fullmatch(prefix) + or (has_instance and instance and not _SYSTEMD_UNIT_PART.fullmatch(instance))): + raise CoverageError("Invalid application unit identifier") + return value + + +def _account_name(value: Any) -> str: + if not isinstance(value, str) or not _ACCOUNT.fullmatch(value): + raise CoverageError("Invalid Linux account name") + return value + + +def _sha(value: Any, description: str) -> str: + if not isinstance(value, str) or not _DIGEST.fullmatch(value): + raise CoverageError(f"Invalid {description} digest") + return value + + +def _absolute(value: Any, description: str) -> Path: + if (not isinstance(value, str) or not value.startswith("/") or "\0" in value + or "\n" in value or "\r" in value or ".." in Path(value).parts): + raise CoverageError(f"Invalid private {description} path") + path = Path(value) + if str(path) != value: + raise CoverageError(f"Invalid private {description} path") + return path + + +def validate_scope_policy(raw: Any) -> dict[str, Any]: + """Validate the portable schema-1 policy before private path resolution.""" + if not isinstance(raw, dict) or set(raw) != { + "schema_version", "approved_roots", "require_privileged", + "filesystem_boundaries", "metadata_expectations", + }: + raise CoverageError("Invalid portable coverage policy schema") + if type(raw["schema_version"]) is not int or raw["schema_version"] != 1: + raise CoverageError("Unsupported portable coverage policy version") + roots = raw["approved_roots"] + if not isinstance(roots, list) or not roots or len(roots) > 64: + raise CoverageError("Portable coverage roots must be a bounded nonempty list") + roots = [_identifier(root, "root") for root in roots] + if len(set(roots)) != len(roots): + raise CoverageError("Duplicate portable coverage root") + if type(raw["require_privileged"]) is not bool or raw["filesystem_boundaries"] != "explicit": + raise CoverageError("Coverage privilege and filesystem boundary policy must be explicit") + expectations = raw["metadata_expectations"] + if (not isinstance(expectations, dict) or set(expectations) != _METADATA + or any(expectations[key] is not True for key in _METADATA)): + raise CoverageError("Full numeric ownership, mode, ACL, xattr and link metadata are required") + return {"schema_version": 1, "approved_roots": roots, + "require_privileged": raw["require_privileged"], + "filesystem_boundaries": "explicit", + "metadata_expectations": {key: True for key in sorted(_METADATA)}} + + +def _read_private(path: Path, limit: int = 1024 * 1024) -> bytes: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077: + raise CoverageError("Coverage input must be an owned private regular file") + content = stream.read(limit + 1) + except OSError: + raise CoverageError("Private coverage input is unavailable") from None + if len(content) > limit: + raise CoverageError("Private coverage input exceeds its size limit") + return content + + +def _recipe(raw: Any) -> dict[str, str]: + if not isinstance(raw, dict) or set(raw) != {"reason", "consequence", "recipe_id", "recipe_sha256"}: + raise CoverageError("Coverage reconstruction recipe is incomplete") + reason, consequence = raw["reason"], raw["consequence"] + if any(not isinstance(value, str) or not value.strip() or len(value) > 256 + or any(char in value for char in "\0\r\n") for value in (reason, consequence)): + raise CoverageError("Coverage exclusion must explain reason and consequence") + return {"reason": reason, "consequence": consequence, + "recipe_id": _identifier(raw["recipe_id"], "recipe"), + "recipe_sha256": _sha(raw["recipe_sha256"], "recipe")} + + +def _declaration(raw: Any, scope: dict[str, Any]) -> dict[str, Any]: + schema1_keys = { + "schema_version", "roots", "exclusions", "mount_decisions", + "identity_decisions", "applications", "package_reconstruction", "reconstruction_evidence", + } + if (not isinstance(raw, dict) or type(raw.get("schema_version")) is not int + or (raw["schema_version"] == 1 and set(raw) != schema1_keys) + or (raw["schema_version"] == 2 and set(raw) != schema1_keys | { + "database_roots", "postgresql_clusters"}) + or raw["schema_version"] not in (1, 2)): + raise CoverageError("Invalid private coverage declaration schema") + roots = [] + if not isinstance(raw["roots"], list) or len(raw["roots"]) > 64: + raise CoverageError("Private coverage roots must match portable scope") + for item in raw["roots"]: + if not isinstance(item, dict) or set(item) != {"id", "path", "source_id", "adapter"}: + raise CoverageError("Invalid private coverage root") + root = {"id": _identifier(item["id"], "root"), + "path": str(_absolute(item["path"], "root")), + "source_id": _identifier(item["source_id"], "source"), + "adapter": item["adapter"]} + if root["adapter"] not in ("files", "sqlite", "postgresql", "mysql", "mariadb"): + raise CoverageError("Unsupported declared capture adapter") + roots.append(root) + if raw["schema_version"] == 2 and any(root["adapter"] not in ("files", "sqlite") + for root in roots): + raise CoverageError("Schema-2 physical roots must use file or SQLite adapters") + if {root["id"] for root in roots} != set(scope["approved_roots"]): + if raw["schema_version"] == 1: + raise CoverageError("Private coverage roots differ from portable scope") + if len({root["path"] for root in roots}) != len(roots): + raise CoverageError("Duplicate private coverage path") + exclusions = [] + if not isinstance(raw["exclusions"], list) or len(raw["exclusions"]) > 256: + raise CoverageError("Invalid coverage exclusions") + for item in raw["exclusions"]: + if not isinstance(item, dict) or set(item) != {"root_id", "path", "recipe"}: + raise CoverageError("Invalid exact coverage exclusion") + root_id = _identifier(item["root_id"], "root") + root_path = next((Path(root["path"]) for root in roots if root["id"] == root_id), None) + path = _absolute(item["path"], "exclusion") + if root_path is None or path == root_path or root_path not in path.parents: + raise CoverageError("Exact exclusion must be inside its declared root") + exclusions.append({"root_id": root_id, "path": str(path), "recipe": _recipe(item["recipe"])}) + if len({item["path"] for item in exclusions}) != len(exclusions): + raise CoverageError("Duplicate exact coverage exclusion") + mounts = [] + if not isinstance(raw["mount_decisions"], list) or len(raw["mount_decisions"]) > 512: + raise CoverageError("Invalid mount decisions") + for item in raw["mount_decisions"]: + if not isinstance(item, dict) or set(item) != {"mount_id", "mountpoint", "decision"}: + raise CoverageError("Invalid exact mount decision") + if type(item["mount_id"]) is not int or item["mount_id"] <= 0 or item["decision"] not in ("include", "exclude"): + raise CoverageError("Invalid exact mount decision") + mounts.append({"mount_id": item["mount_id"], + "mountpoint": str(_absolute(item["mountpoint"], "mountpoint")), + "decision": item["decision"]}) + if len({item["mountpoint"] for item in mounts}) != len(mounts): + raise CoverageError("Duplicate mount decision") + identities = [] + if not isinstance(raw["identity_decisions"], list) or len(raw["identity_decisions"]) > 4096: + raise CoverageError("Invalid identity reconstruction decisions") + for item in raw["identity_decisions"]: + if not isinstance(item, dict) or set(item) != {"name", "home", "recipe"}: + raise CoverageError("Invalid identity reconstruction decision") + identities.append({"name": _account_name(item["name"]), + "home": str(_absolute(item["home"], "home")), + "recipe": _recipe(item["recipe"])}) + if len({item["name"] for item in identities}) != len(identities): + raise CoverageError("Duplicate identity decision") + applications = [] + if not isinstance(raw["applications"], list) or len(raw["applications"]) > 512: + raise CoverageError("Invalid application declarations") + for item in raw["applications"]: + if not isinstance(item, dict) or set(item) != {"id", "unit", "source_id", "state_roots", "startup_recipe"}: + raise CoverageError("Invalid application declaration") + if not isinstance(item["state_roots"], list) or len(item["state_roots"]) > 64: + raise CoverageError("Invalid application state roots") + applications.append({"id": _identifier(item["id"], "application"), + "unit": _systemd_service_stem(item["unit"]) + ".service", + "source_id": _identifier(item["source_id"], "source"), + "state_roots": [str(_absolute(path, "application state")) for path in item["state_roots"]], + "startup_recipe": _recipe(item["startup_recipe"])}) + if len({item["id"] for item in applications}) != len(applications): + raise CoverageError("Duplicate application declaration") + package_recipe = raw["package_reconstruction"] + if package_recipe is not None: + package_recipe = _recipe(package_recipe) + evidence = [] + if not isinstance(raw["reconstruction_evidence"], list) or len(raw["reconstruction_evidence"]) > 512: + raise CoverageError("Invalid reconstruction evidence references") + for item in raw["reconstruction_evidence"]: + if not isinstance(item, dict) or set(item) != { + "recipe_id", "recipe_sha256", "procedure_path", "evidence_path", "evidence_sha256"}: + raise CoverageError("Invalid reconstruction evidence reference") + evidence.append({"recipe_id": _identifier(item["recipe_id"], "recipe"), + "recipe_sha256": _sha(item["recipe_sha256"], "recipe"), + "procedure_path": str(_absolute(item["procedure_path"], "procedure")), + "evidence_path": str(_absolute(item["evidence_path"], "evidence")), + "evidence_sha256": _sha(item["evidence_sha256"], "evidence")}) + if len({(item["recipe_id"], item["recipe_sha256"]) for item in evidence}) != len(evidence): + raise CoverageError("Duplicate reconstruction evidence reference") + database_roots = [] + clusters = [] + if raw["schema_version"] == 2: + if (not isinstance(raw["database_roots"], list) or not raw["database_roots"] + or len(raw["database_roots"]) > 64 + or not isinstance(raw["postgresql_clusters"], list) + or len(raw["postgresql_clusters"]) != 1): + raise CoverageError("PostgreSQL coverage requires one bounded cluster") + for item in raw["database_roots"]: + if not isinstance(item, dict) or set(item) != { + "id", "adapter", "source_id", "cluster_id", "connection", "database"}: + raise CoverageError("Invalid logical PostgreSQL database root") + database = item["database"] + if (item["adapter"] != "postgresql" or not isinstance(database, str) + or not database or len(database) > 128 or any(c in database for c in "\0\r\n")): + raise CoverageError("Invalid logical PostgreSQL database root") + database_roots.append({"id": _identifier(item["id"], "root"), + "adapter": "postgresql", + "source_id": _identifier(item["source_id"], "source"), + "cluster_id": _identifier(item["cluster_id"], "cluster"), + "connection": _identifier(item["connection"], "connection"), + "database": database}) + for item in raw["postgresql_clusters"]: + if not isinstance(item, dict) or set(item) != { + "id", "connection", "data_directory", "databases", "reconstruction", + "consistency_window"}: + raise CoverageError("Invalid PostgreSQL cluster declaration") + if item["consistency_window"] != {"quiet_administrative_window": True, + "schema_changes_frozen": True}: + raise CoverageError("PostgreSQL consistency window is not explicitly attested") + if not isinstance(item["databases"], list) or not item["databases"] or len(item["databases"]) > 4096: + raise CoverageError("PostgreSQL database decisions must be complete and bounded") + cluster_id = _identifier(item["id"], "cluster") + connection = _identifier(item["connection"], "connection") + data_directory = str(_absolute(item["data_directory"], "PostgreSQL data directory")) + decisions = [] + for decision in item["databases"]: + if (not isinstance(decision, dict) or decision.get("decision") not in ("capture", "reconstruct") + or set(decision) != ({"name", "oid", "decision", "root_id"} + if decision.get("decision") == "capture" + else {"name", "oid", "decision"})): + raise CoverageError("Invalid PostgreSQL database reconstruction decision") + name, oid = decision["name"], decision["oid"] + if (not isinstance(name, str) or not name or len(name) > 128 + or any(c in name for c in "\0\r\n") or type(oid) is not int or oid <= 0): + raise CoverageError("Invalid PostgreSQL database identity") + clean = {"name": name, "oid": oid, "decision": decision["decision"]} + if decision["decision"] == "capture": + clean["root_id"] = _identifier(decision["root_id"], "root") + decisions.append(clean) + if (len({choice["name"] for choice in decisions}) != len(decisions) + or len({choice["oid"] for choice in decisions}) != len(decisions)): + raise CoverageError("Duplicate PostgreSQL database decision") + reconstruction = item["reconstruction"] + if not isinstance(reconstruction, dict) or set(reconstruction) != { + "recipe", "procedure_path", "evidence_path", "evidence_sha256", "recovery_material"}: + raise CoverageError("PostgreSQL reconstruction evidence is incomplete") + material = reconstruction["recovery_material"] + if not isinstance(material, list) or not material or len(material) > 128: + raise CoverageError("PostgreSQL recovery material references must be bounded") + material_clean = [] + for ref in material: + if not isinstance(ref, dict) or set(ref) != {"path", "sha256"}: + raise CoverageError("Invalid PostgreSQL recovery material reference") + material_clean.append({"path": str(_absolute(ref["path"], "recovery material")), + "sha256": _sha(ref["sha256"], "recovery material")}) + if len({ref["path"] for ref in material_clean}) != len(material_clean): + raise CoverageError("Duplicate PostgreSQL recovery material reference") + clusters.append({"id": cluster_id, "connection": connection, + "data_directory": data_directory, "databases": decisions, + "consistency_window": item["consistency_window"], + "reconstruction": {"recipe": _recipe(reconstruction["recipe"]), + "procedure_path": str(_absolute(reconstruction["procedure_path"], "procedure")), + "evidence_path": str(_absolute(reconstruction["evidence_path"], "evidence")), + "evidence_sha256": _sha(reconstruction["evidence_sha256"], "evidence"), + "recovery_material": material_clean}}) + if len({root["id"] for root in database_roots} | {root["id"] for root in roots}) != len(database_roots) + len(roots): + raise CoverageError("Duplicate physical and logical coverage root") + if {root["id"] for root in database_roots} | {root["id"] for root in roots} != set(scope["approved_roots"]): + raise CoverageError("Private coverage roots differ from portable scope") + cluster = clusters[0] + if cluster["connection"] != database_roots[0]["connection"]: + raise CoverageError("PostgreSQL root and cluster selectors differ") + if any(root["cluster_id"] != cluster["id"] or root["connection"] != cluster["connection"] + for root in database_roots): + raise CoverageError("PostgreSQL database roots must select the declared local cluster") + capture_ids = {choice["root_id"] for choice in cluster["databases"] + if choice["decision"] == "capture"} + if capture_ids != {root["id"] for root in database_roots}: + raise CoverageError("PostgreSQL capture decisions differ from logical roots") + declaration = {"schema_version": raw["schema_version"], "roots": roots, + "exclusions": exclusions, "mount_decisions": mounts, + "identity_decisions": identities, "applications": applications, + "package_reconstruction": package_recipe, + "reconstruction_evidence": evidence} + if raw["schema_version"] == 2: + declaration["database_roots"] = database_roots + declaration["postgresql_clusters"] = clusters + return declaration + + +def _bounded_read(path: Path, limit: int = _MAX_INVENTORY) -> bytes: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise CoverageError("Linux inventory source is not regular") + content = stream.read(limit + 1) + except OSError: + raise CoverageError("Linux inventory source is unavailable") from None + if len(content) > limit: + raise CoverageError("Linux inventory exceeds its size limit") + return content + + +def _mount_unescape(value: str) -> str: + def replace(match): + return chr(int(match.group(1), 8)) + return re.sub(r"\\([0-7]{3})", replace, value) + + +def _mount_inventory(raw: bytes) -> list[dict[str, Any]]: + mounts = [] + try: + for line in raw.decode("utf-8").splitlines(): + before, after = line.split(" - ", 1) + fields, tail = before.split(), after.split() + if len(fields) < 6 or len(tail) < 3: + raise ValueError() + mount_id, parent_id = int(fields[0]), int(fields[1]) + if mount_id <= 0 or parent_id <= 0: + raise ValueError() + mounts.append({"mount_id": mount_id, "parent_id": parent_id, + "root": _mount_unescape(fields[3]), + "mountpoint": _mount_unescape(fields[4]), + "filesystem": tail[0], "source_digest": hashlib.sha256(tail[1].encode()).hexdigest()}) + except (UnicodeError, ValueError): + raise CoverageError("Linux mount inventory is malformed") from None + if len(mounts) > 8192 or len({item["mount_id"] for item in mounts}) != len(mounts): + raise CoverageError("Linux mount inventory is ambiguous") + return sorted(mounts, key=lambda item: item["mount_id"]) + + +def _accounts(raw: bytes, *, passwd: bool) -> list[dict[str, Any]]: + accounts = [] + try: + for line in raw.decode("utf-8").splitlines(): + if not line or line.startswith("#"): + continue + fields = line.split(":") + if len(fields) != (7 if passwd else 4): + raise ValueError() + name = _account_name(fields[0]) + uid = int(fields[2]) + if uid < 0: + raise ValueError() + account = {"name": name, "id": uid} + if passwd: + account["gid"] = int(fields[3]) + account["home"] = fields[5] + account["shell"] = fields[6] + else: + account["members"] = fields[3].split(",") if fields[3] else [] + accounts.append(account) + except (UnicodeError, ValueError, CoverageError): + raise CoverageError("Linux account inventory is malformed") from None + if len(accounts) > 65536 or len({item["name"] for item in accounts}) != len(accounts): + raise CoverageError("Linux account inventory is ambiguous") + return sorted(accounts, key=lambda item: item["name"]) + + +def _application_units(directory: Path) -> list[dict[str, str]]: + try: + if not stat.S_ISDIR(directory.lstat().st_mode): + raise CoverageError("Application unit inventory is not a directory") + except OSError: + raise CoverageError("Application unit inventory is unavailable") from None + stack = [(directory, 0)] + units = [] + visited = 0 + while stack: + path, depth = stack.pop() + if depth > 16 or visited > 8192: + raise CoverageError("Application unit inventory exceeds its size limit") + try: + with os.scandir(path) as entries: + children = sorted((Path(entry.path) for entry in entries), reverse=True) + except OSError: + raise CoverageError("Application unit inventory is unavailable") from None + for child in children: + visited += 1 + if visited > 8192: + raise CoverageError("Application unit inventory exceeds its size limit") + try: + info = child.lstat() + except OSError: + raise CoverageError("Application unit inventory is unavailable") from None + if stat.S_ISDIR(info.st_mode): + stack.append((child, depth + 1)) + elif child.name.endswith(".service"): + if stat.S_ISLNK(info.st_mode): + try: + target = os.readlink(child) + except OSError: + raise CoverageError("Application unit link is unavailable") from None + kind = "link:" + hashlib.sha256(target.encode("utf-8", "surrogateescape")).hexdigest() + elif stat.S_ISREG(info.st_mode): + kind = "regular" + else: + raise CoverageError("Application unit type is unsupported") + units.append({"name": child.name, "relative_path": str(child.relative_to(directory)), + "kind": kind}) + return sorted(units, key=lambda item: item["relative_path"]) + + +def _within(path: Path, root: Path) -> bool: + return path == root or root in path.parents + + +def sqlite_file_identity(path: Path) -> dict[str, Any]: + """Read stable file identity and reconstructable metadata for a SQLite file.""" + path = Path(path) + if not path.is_absolute() or ".." in path.parts or _has_symlink_ancestor(path): + raise _SQLiteCoverageError("sqlite_database_unsafe") + try: + before = path.lstat() + except FileNotFoundError: + raise _SQLiteCoverageError("sqlite_database_missing") from None + except OSError: + raise _SQLiteCoverageError("sqlite_database_unavailable") from None + if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1: + raise _SQLiteCoverageError("sqlite_database_unsafe") + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + try: + descriptor = os.open(path, flags) + except PermissionError: + raise _SQLiteCoverageError("sqlite_database_unreadable") from None + except OSError: + raise _SQLiteCoverageError("sqlite_database_unavailable") from None + try: + info = os.fstat(descriptor) + if (not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 + or (info.st_dev, info.st_ino) != (before.st_dev, before.st_ino) + or _has_symlink_ancestor(path)): + raise _SQLiteCoverageError("sqlite_database_unsafe") + try: + names = sorted(os.listxattr(descriptor)) + if len(names) > _MAX_SQLITE_XATTRS: + raise OSError("xattr count exceeds limit") + xattrs = [] + total = 0 + for name in names: + value = os.getxattr(descriptor, name) + total += len(name.encode("utf-8", "surrogateescape")) + len(value) + if total > _MAX_SQLITE_XATTR_BYTES: + raise OSError("xattr bytes exceed limit") + xattrs.append({"name": name, "value_base64": base64.b64encode(value).decode("ascii")}) + except OSError: + raise _SQLiteCoverageError("sqlite_metadata_unreadable") from None + for suffix in ("-wal", "-shm", "-journal"): + sidecar = Path(str(path) + suffix) + try: + sidecar_info = sidecar.lstat() + except FileNotFoundError: + continue + except OSError: + raise _SQLiteCoverageError("sqlite_sidecar_unsafe") from None + if (not stat.S_ISREG(sidecar_info.st_mode) or sidecar_info.st_nlink != 1 + or _has_symlink_ancestor(sidecar)): + raise _SQLiteCoverageError("sqlite_sidecar_unsafe") + identity = {"device": info.st_dev, "inode": info.st_ino, + "uid": info.st_uid, "gid": info.st_gid, + "mode": info.st_mode & 0o7777, "links": info.st_nlink, + "xattrs": xattrs} + return identity + finally: + os.close(descriptor) + + +def verify_sqlite_identity(path: Path, expected: dict[str, Any]) -> None: + """Fail closed if reviewed SQLite identity or metadata changed.""" + if sqlite_file_identity(path) != expected: + raise _SQLiteCoverageError("sqlite_reviewed_identity_changed") + + +def _sqlite_identity_with_integrity(path: Path) -> dict[str, Any]: + identity = sqlite_file_identity(path) + try: + with sqlite3.connect(path.as_uri() + "?mode=ro", uri=True, timeout=2) as database: + if database.execute("PRAGMA quick_check").fetchall() != [("ok",)]: + raise _SQLiteCoverageError("sqlite_database_invalid") + except sqlite3.Error: + raise _SQLiteCoverageError("sqlite_database_invalid") from None + return identity + + +def _has_symlink_ancestor(path: Path) -> bool: + current = Path("/") + for part in path.parts[1:]: + current /= part + try: + if stat.S_ISLNK(current.lstat().st_mode): + return True + except OSError: + return False + return False + + +def _scope_location(path: Path, roots: dict[str, Path], exclusions: set[Path]) -> str: + """Classify a logical state path by the bytes that capture will actually see.""" + if any(_within(path, exclusion) for exclusion in exclusions): + return "excluded" + if not any(_within(path, root) for root in roots.values()): + return "external" + if _has_symlink_ancestor(path): + try: + target = path.resolve(strict=True) + except (OSError, RuntimeError): + return "linked" + if any(_within(target, exclusion) for exclusion in exclusions): + return "linked" + if not any(_within(target, root) for root in roots.values()): + return "linked" + return "captured" + + +def restic_literal_exclusion(path: Path) -> str: + """Escape Restic glob metacharacters in one reviewed absolute path.""" + if not path.is_absolute(): + raise CoverageError("Coverage exclusion must be absolute") + return "".join("\\" + char if char in "*?[]\\" else char for char in str(path)) + + +def _evidence_reasons(declaration: dict[str, Any]) -> set[str]: + required = [item["recipe"] for item in declaration["exclusions"]] + required.extend(item["recipe"] for item in declaration["identity_decisions"]) + required.extend(item["startup_recipe"] for item in declaration["applications"]) + if declaration["package_reconstruction"] is not None: + required.append(declaration["package_reconstruction"]) + references = {(item["recipe_id"], item["recipe_sha256"]): item + for item in declaration["reconstruction_evidence"]} + reasons = set() + for recipe in required: + key = (recipe["recipe_id"], recipe["recipe_sha256"]) + reference = references.get(key) + if reference is None: + reasons.add("reconstruction_evidence_missing") + continue + try: + procedure = _read_private(Path(reference["procedure_path"])) + evidence_bytes = _read_private(Path(reference["evidence_path"])) + record = strict_json(evidence_bytes.decode("utf-8")) + except (CoverageError, ConfigurationError, UnicodeError): + reasons.add("reconstruction_evidence_unverified") + continue + if (hashlib.sha256(procedure).hexdigest() != recipe["recipe_sha256"] + or hashlib.sha256(evidence_bytes).hexdigest() != reference["evidence_sha256"] + or not isinstance(record, dict) or set(record) != { + "schema_version", "recipe_id", "recipe_sha256", "environment", "checks"} + or record["schema_version"] != 1 + or record["recipe_id"] != recipe["recipe_id"] + or record["recipe_sha256"] != recipe["recipe_sha256"] + or record["environment"] not in ("isolated-local-fixture", "reviewed-local-procedure") + or record["checks"] != {"procedure_executed": True, "result_verified": True}): + reasons.add("reconstruction_evidence_unverified") + return reasons + + +def _walk(root: Path, exclusions: set[Path], mounts: dict[Path, dict], decisions: dict[Path, str], + reasons: set[str], counts: dict[str, int], metadata: "hashlib._Hash", + links: dict[tuple[int, int], tuple[int, int]], skipped: set[Path] | None = None) -> None: + stack = [(root, 0)] + skipped = skipped or set() + while stack: + path, depth = stack.pop() + if path in skipped: + continue + if path in exclusions: + counts["excluded"] += 1 + continue + if depth > 128 or counts["entries"] >= _MAX_ENTRIES: + reasons.add("inventory_limit_reached") + return + mount = mounts.get(path) + if mount is not None: + choice = decisions.get(path) + if choice is None: + reasons.add("mount_boundary_undecided") + continue + if choice == "exclude": + reasons.add("mount_exclusion_requires_exact_recipe") + continue + try: + info = path.lstat() + except OSError: + reasons.add("unreadable_entry") + continue + counts["entries"] += 1 + mode = stat.S_IFMT(info.st_mode) + if not (stat.S_ISREG(info.st_mode) or stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)): + reasons.add("unsupported_entry_type") + continue + try: + attrs = sorted(os.listxattr(path, follow_symlinks=False)) + if len(attrs) > 256: + raise OSError("too many metadata attributes") + attr_digests = [] + for name in attrs: + value = os.getxattr(path, name, follow_symlinks=False) + if len(value) > 1024 * 1024: + raise OSError("metadata attribute too large") + attr_digests.append((name, hashlib.sha256(value).hexdigest())) + except OSError: + reasons.add("metadata_unreadable") + attr_digests = [] + metadata.update(_canonical([str(path), info.st_uid, info.st_gid, info.st_mode & 0o7777, + info.st_nlink, mode, attr_digests])) + if stat.S_ISREG(info.st_mode) and info.st_nlink > 1: + pair = (info.st_dev, info.st_ino) + observed, expected = links.get(pair, (0, info.st_nlink)) + links[pair] = (observed + 1, expected) + if stat.S_ISLNK(info.st_mode): + counts["symlinks"] += 1 + continue + if stat.S_ISREG(info.st_mode): + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != (info.st_dev, info.st_ino): + raise OSError("inventory changed") + finally: + os.close(descriptor) + except OSError: + reasons.add("unreadable_file") + counts["files"] += 1 + continue + counts["directories"] += 1 + try: + with os.scandir(path) as entries: + children = sorted((Path(entry.path) for entry in entries), reverse=True) + except OSError: + reasons.add("unreadable_directory") + continue + stack.extend((child, depth + 1) for child in children) +def _scan(declaration: dict[str, Any], scope: dict[str, Any], *, inventory: InventoryPaths, + policy_bytes: bytes, binding_bytes: bytes, enforce_mount_ids: bool = True, + postgresql_context: PostgreSQLCoverageContext | None = None) -> dict[str, Any]: + reasons: set[str] = set() + if scope["require_privileged"] and os.geteuid() != 0: + reasons.add("privileged_inventory_required") + users = _accounts(_bounded_read(inventory.passwd), passwd=True) + groups = _accounts(_bounded_read(inventory.group), passwd=False) + mount_inventory = _mount_inventory(_bounded_read(inventory.mountinfo)) + package_digest = hashlib.sha256(_bounded_read(inventory.packages)).hexdigest() + units = _application_units(inventory.application_units) + roots = {item["id"]: Path(item["path"]) for item in declaration["roots"]} + postgresql_data_directories = [Path(item["data_directory"]) + for item in declaration.get("postgresql_clusters", [])] + # Only mounts whose data boundary can affect a reviewed root belong in + # drift identity. systemd PrivateTmp and other unrelated namespace mounts + # may have new kernel mount IDs on every invocation. + mounts = [item for item in mount_inventory if any( + _within(Path(item["mountpoint"]), root) or _within(root, Path(item["mountpoint"])) + for root in roots.values()) and not any( + _within(Path(item["mountpoint"]), data_directory) + for data_directory in postgresql_data_directories)] + postgresql_storage_mounts = [ + {key: item[key] for key in ("root", "mountpoint", "filesystem", "source_digest")} + for item in mount_inventory if any( + _within(Path(item["mountpoint"]), data_directory) + for data_directory in postgresql_data_directories)] + if any(Path(item["mountpoint"]) != data_directory + for item in postgresql_storage_mounts for data_directory in postgresql_data_directories + if _within(Path(item["mountpoint"]), data_directory)): + reasons.add("postgresql_external_storage_mount_unsupported") + stable_mounts = sorted(({key: item[key] for key in + ("root", "mountpoint", "filesystem", "source_digest")} + for item in mounts), + key=lambda item: (item["mountpoint"], item["root"], + item["filesystem"], item["source_digest"])) + if any(item["adapter"] not in ("files", "sqlite") for item in declaration["roots"]): + reasons.add("capture_adapter_unsupported_for_coverage") + exclusions = {Path(item["path"]) for item in declaration["exclusions"]} + reasons.update(_evidence_reasons(declaration)) + postgresql = {"reasons": [], "approved_postgresql": [], + "capture_exclusions": [], "clusters": []} + if declaration["schema_version"] == 2: + postgresql = inspect_postgresql_coverage(declaration["database_roots"], + declaration["postgresql_clusters"], + postgresql_context, declaration["roots"], + declaration["exclusions"]) + reasons.update(postgresql["reasons"]) + postgresql_skip_paths: set[Path] = set() + postgresql_capture_exclusions: set[Path] = set() + for raw_directory in postgresql["capture_exclusions"]: + data_directory = Path(raw_directory) + overlaps = [item for item in declaration["roots"] + if _within(Path(item["path"]), data_directory) + or _within(data_directory, Path(item["path"]))] + unsafe_overlap = [item for item in overlaps + if item["adapter"] != "files" + or _within(Path(item["path"]), data_directory)] + if unsafe_overlap: + reasons.add("postgresql_raw_storage_root_conflict") + continue + postgresql_skip_paths.add(data_directory) + postgresql_capture_exclusions.add(data_directory) + sqlite_roots = [item for item in declaration["roots"] if item["adapter"] == "sqlite"] + sqlite_identities: dict[str, dict[str, Any]] = {} + sqlite_recipes: dict[str, dict[str, str]] = {} + sqlite_skip_paths: set[Path] = set() + sqlite_capture_exclusions: set[Path] = set() + for root in sqlite_roots: + path = Path(root["path"]) + try: + identity = _sqlite_identity_with_integrity(path) + sqlite_identities[root["id"]] = identity + except _SQLiteCoverageError as exc: + reasons.add(exc.reason) + for exclusion in exclusions: + if _within(path, exclusion) or _within(exclusion, path): + reasons.add("sqlite_state_excluded") + exact_apps = [app for app in declaration["applications"] + if app["source_id"] == root["source_id"] + and app["state_roots"] == [str(path)]] + overlapping_apps = [app for app in declaration["applications"] + if str(path) in app["state_roots"]] + if len(exact_apps) != 1 or len(overlapping_apps) != 1: + reasons.add("sqlite_application_state_unaccounted") + else: + sqlite_recipes[root["id"]] = exact_apps[0]["startup_recipe"] + paths = {path, *(Path(str(path) + suffix) for suffix in ("-wal", "-shm", "-journal"))} + sqlite_skip_paths.update(paths) + sqlite_capture_exclusions.update(paths) + if any(item["adapter"] == "files" and Path(item["path"]) in paths + for item in declaration["roots"]): + reasons.add("sqlite_artifact_explicitly_mapped") + for exclusion in declaration["exclusions"]: + path = Path(exclusion["path"]) + root = next(item for item in declaration["roots"] if item["id"] == exclusion["root_id"]) + if root["adapter"] != "files" or not Path(root["path"]).is_dir(): + reasons.add("exclusion_root_not_directory") + if _has_symlink_ancestor(path.parent): + reasons.add("exclusion_linked_path") + try: + path.lstat() + except OSError: + reasons.add("exclusion_target_missing") + decisions = {Path(item["mountpoint"]): item["decision"] + for item in declaration["mount_decisions"]} + mount_by_path = {Path(item["mountpoint"]): item for item in mounts} + if len(mount_by_path) != len(mounts): + reasons.add("mount_inventory_ambiguous") + for decision in declaration["mount_decisions"]: + mount = mount_by_path.get(Path(decision["mountpoint"])) + if mount is None or (enforce_mount_ids and mount["mount_id"] != decision["mount_id"]): + reasons.add("mount_decision_stale") + if decision["decision"] == "exclude" and Path(decision["mountpoint"]) not in exclusions: + reasons.add("mount_exclusion_requires_exact_recipe") + for mount in mounts: + mountpoint = Path(mount["mountpoint"]) + if any(_within(mountpoint, root) for root in roots.values()) and mountpoint not in decisions: + reasons.add("mount_boundary_undecided") + identities = {(item["name"], item["home"]) for item in declaration["identity_decisions"]} + for user in users: + home = Path(user["home"]) + location = _scope_location(home, roots, exclusions) if home.is_absolute() else "external" + if location != "captured" and (user["name"], user["home"]) not in identities: + reasons.add("user_home_unaccounted") + app_units = {item["unit"] for item in declaration["applications"]} + if {item["name"] for item in units} - app_units: + reasons.add("application_unaccounted") + for app in declaration["applications"]: + if app["unit"] not in {item["name"] for item in units}: + reasons.add("application_unit_missing") + for state_root in app["state_roots"]: + location = _scope_location(Path(state_root), roots, exclusions) + if (location == "captured" and not any( + _within(Path(state_root), Path(root["path"])) and app["source_id"] == root["source_id"] + for root in declaration["roots"])) or location in ("external", "linked"): + reasons.add("application_state_unaccounted") + if not any(_within(inventory.packages, root) for root in roots.values()) and declaration["package_reconstruction"] is None: + reasons.add("package_reinstall_unaccounted") + database_roots = declaration.get("database_roots", []) + counts = {"roots": len(roots) + len(database_roots), "entries": 0, "files": 0, "directories": 0, + "symlinks": 0, "excluded": 0, "users": len(users), "groups": len(groups), + "mounts": len(mounts), "applications": len(units), + "sqlite_databases": len(sqlite_roots)} + if declaration["schema_version"] == 2: + counts["postgresql_databases"] = len(database_roots) + metadata = hashlib.sha256() + links: dict[tuple[int, int], tuple[int, int]] = {} + for item in declaration["roots"]: + root = Path(item["path"]) + try: + if _has_symlink_ancestor(root): + reasons.add("root_symlink_unsupported") + continue + except OSError: + reasons.add("root_unreadable") + continue + if item["adapter"] == "sqlite": + identity = sqlite_identities.get(item["id"]) + if identity is not None: + counts["entries"] += 1 + counts["files"] += 1 + metadata.update(_canonical(["sqlite", str(root), identity])) + continue + _walk(root, exclusions, mount_by_path, decisions, reasons, counts, metadata, links, + skipped=sqlite_skip_paths | postgresql_skip_paths) + if any(observed < expected for observed, expected in links.values()): + reasons.add("hardlink_scope_unproven") + structural = {"users": users, "groups": groups, "mounts": stable_mounts, + "applications": units, "declared_applications": declaration["applications"], + "roots": [{"id": item["id"], "path": item["path"], "source_id": item["source_id"], + "adapter": item["adapter"], "exists": Path(item["path"]).exists(), + **({"sqlite_identity": sqlite_identities.get(item["id"])} + if item["adapter"] == "sqlite" else {})} + for item in declaration["roots"]], + "package_digest": package_digest} + if declaration["schema_version"] == 2: + structural["database_roots"] = database_roots + structural["postgresql"] = postgresql["clusters"] + structural["postgresql_storage_mounts"] = sorted(postgresql_storage_mounts, + key=lambda item: (item["mountpoint"], item["root"], item["filesystem"], item["source_digest"])) + approved_sqlite = [{"root_id": root["id"], "path": root["path"], + "source_id": root["source_id"], + "identity": sqlite_identities.get(root["id"]), + "recipe_digest": sqlite_recipes.get(root["id"], {}).get("recipe_sha256")} + for root in sqlite_roots] + candidate = {"schema_version": 1, "declaration_digest": _digest(declaration), + "scope_policy_digest": _digest(scope), "enrollment": evidence_bytes(policy_bytes, binding_bytes), + "structural_digest": _digest(structural), "metadata_digest": metadata.hexdigest(), + "counts": counts, "reasons": sorted(reasons), "roots": declaration["roots"], + "approved_sqlite": approved_sqlite, + "sqlite_capture_exclusions": sorted(map(str, sqlite_capture_exclusions))} + if declaration["schema_version"] == 2: + candidate["approved_postgresql"] = postgresql["approved_postgresql"] + candidate["postgresql_capture_exclusions"] = sorted(map(str, postgresql_capture_exclusions)) + return candidate + + +def _private_directory(state: Path, *, create: bool = True) -> Path: + base = Path(state) / "coverage" + try: + if create: + base.mkdir(mode=0o700, exist_ok=True) + info = base.lstat() + except OSError: + raise CoverageError("Private coverage state is unavailable") from None + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077: + raise CoverageError("Private coverage state is unsafe") + return base + + +def _private_child(base: Path, name: str, *, create: bool) -> Path: + child = base / name + try: + if create: + child.mkdir(mode=0o700, exist_ok=True) + info = child.lstat() + except OSError: + raise CoverageError("Private coverage record directory is unavailable") from None + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077: + raise CoverageError("Private coverage record directory is unsafe") + return child + + +def _active_directory(base: Path, declaration_path: Path, scope: dict[str, Any], *, create: bool) -> Path: + profiles = _private_child(base, "profiles", create=create) + key = _digest({"scope_policy_digest": _digest(scope), + "declaration_path": str(Path(declaration_path))}) + return _private_child(profiles, key, create=create) + + +def _write_immutable(directory: Path, value: dict[str, Any]) -> str: + digest = _digest(value) + target = directory / f"{digest}.json" + descriptor, name = tempfile.mkstemp(prefix=".coverage-", dir=directory) + temporary = Path(name) + try: + with os.fdopen(descriptor, "wb") as output: + os.fchmod(output.fileno(), 0o600) + output.write(_canonical(value)) + output.flush() + os.fsync(output.fileno()) + try: + os.link(temporary, target) + except FileExistsError: + if hashlib.sha256(_read_private(target)).hexdigest() != digest: + raise CoverageError("Immutable coverage record was altered") from None + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + finally: + temporary.unlink(missing_ok=True) + return digest + + +def _inputs(declaration_path: Path, scope_policy: Any): + scope = validate_scope_policy(scope_policy) + try: + declaration = _declaration(strict_json(_read_private(Path(declaration_path)).decode("utf-8")), scope) + except (UnicodeError, ConfigurationError): + raise CoverageError("Invalid private coverage declaration JSON") from None + return declaration, scope + + +def discover_coverage(declaration_path: Path, scope_policy: Any, *, state: Path, + policy_bytes: bytes, binding_bytes: bytes, + inventory: InventoryPaths = InventoryPaths(), + postgresql_context: PostgreSQLCoverageContext | None = None) -> dict[str, object]: + """Inventory bounded private scope and persist an immutable review candidate.""" + verify_bytes(state, policy_bytes, binding_bytes) + declaration, scope = _inputs(declaration_path, scope_policy) + candidate = _scan(declaration, scope, inventory=inventory, + policy_bytes=policy_bytes, binding_bytes=binding_bytes, + postgresql_context=postgresql_context) + base = _private_directory(state) + candidates = _private_child(base, "candidates", create=True) + digest = _write_immutable(candidates, candidate) + return {"coverage_status": "complete" if not candidate["reasons"] else "incomplete", + "recovery_status": "untested", "candidate_digest": digest, + "declaration_digest": candidate["declaration_digest"], + "reasons": candidate["reasons"], "counts": candidate["counts"], + "next_action": "Review the exact candidate" if not candidate["reasons"] + else "Resolve inventory gaps and discover again"} + + +def review_coverage(declaration_path: Path, scope_policy: Any, candidate_digest: str, *, state: Path, + policy_bytes: bytes, binding_bytes: bytes, + inventory: InventoryPaths = InventoryPaths(), + postgresql_context: PostgreSQLCoverageContext | None = None) -> dict[str, object]: + """Approve only a complete, exact current candidate under the policy lease.""" + _sha(candidate_digest, "candidate") + with policy_lease(state, exclusive=True): + verify_bytes(state, policy_bytes, binding_bytes) + declaration, scope = _inputs(declaration_path, scope_policy) + base = _private_directory(state) + candidate_path = _private_child(base, "candidates", create=False) / f"{candidate_digest}.json" + try: + recorded = strict_json(_read_private(candidate_path).decode("utf-8")) + except (UnicodeError, ConfigurationError): + raise CoverageError("Reviewed coverage candidate is invalid") from None + current = _scan(declaration, scope, inventory=inventory, + policy_bytes=policy_bytes, binding_bytes=binding_bytes, + postgresql_context=postgresql_context) + if _digest(recorded) != candidate_digest or _digest(current) != candidate_digest: + raise CoverageError("Coverage candidate changed; discover and review again") + if current["reasons"]: + raise CoverageError("Coverage candidate has unresolved inventory gaps") + baseline = {"schema_version": 1, "candidate_digest": candidate_digest, + "structural_digest": current["structural_digest"], + "declaration_digest": current["declaration_digest"], + "scope_policy_digest": current["scope_policy_digest"], + "reviewed_at": datetime.now(timezone.utc).isoformat(timespec="seconds")} + baselines = _private_child(base, "baselines", create=True) + baseline_digest = _write_immutable(baselines, baseline) + active = coverage_evidence_bytes(policy_bytes, binding_bytes, + declaration_digest=current["declaration_digest"], + scope_policy_digest=current["scope_policy_digest"], baseline_digest=baseline_digest) + active["candidate_digest"] = candidate_digest + active["structural_digest"] = current["structural_digest"] + active_dir = _active_directory(base, declaration_path, scope, create=True) + descriptor, name = tempfile.mkstemp(prefix=".coverage-active-", dir=active_dir) + temporary = Path(name) + try: + with os.fdopen(descriptor, "wb") as output: + os.fchmod(output.fileno(), 0o600) + output.write(_canonical(active)) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, active_dir / "active.json") + fd = os.open(active_dir, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + finally: + temporary.unlink(missing_ok=True) + return {"coverage_status": "complete", "recovery_status": "untested", + "baseline_digest": baseline_digest, "candidate_digest": candidate_digest, + "next_action": "Run coverage check before capture"} + + +def check_coverage(declaration_path: Path, scope_policy: Any, *, state: Path, + policy_bytes: bytes, binding_bytes: bytes, + inventory: InventoryPaths = InventoryPaths(), + postgresql_context: PostgreSQLCoverageContext | None = None) -> CoverageCheck: + """Verify enrollment, immutable baseline and current structural coverage.""" + try: + verify_bytes(state, policy_bytes, binding_bytes) + declaration, scope = _inputs(declaration_path, scope_policy) + base = _private_directory(state, create=False) + active_dir = _active_directory(base, declaration_path, scope, create=False) + active = strict_json(_read_private(active_dir / "active.json").decode("utf-8")) + if not isinstance(active, dict) or set(active) != { + "schema_version", "machine_id_digest", "euid", "policy_digest", "bindings_digest", + "declaration_digest", "scope_policy_digest", "baseline_digest", + "candidate_digest", "structural_digest"}: + raise CoverageError("Coverage active record is invalid") + baseline_digest = _sha(active["baseline_digest"], "baseline") + baseline = strict_json(_read_private(_private_child(base, "baselines", create=False) / + f"{baseline_digest}.json").decode("utf-8")) + if (not isinstance(baseline, dict) + or set(baseline) != {"schema_version", "candidate_digest", "structural_digest", + "declaration_digest", "scope_policy_digest", "reviewed_at"} + or baseline.get("schema_version") != 1 + or _digest(baseline) != baseline_digest + or active.get("candidate_digest") != baseline.get("candidate_digest") + or active.get("structural_digest") != baseline.get("structural_digest") + or active.get("declaration_digest") != baseline.get("declaration_digest") + or active.get("scope_policy_digest") != baseline.get("scope_policy_digest")): + raise CoverageError("Coverage baseline was altered") + except (ConfigurationError, CoverageError, OSError, UnicodeError, KeyError): + return CoverageCheck("unenrolled", ("coverage_enrollment_missing_or_invalid",), None, None, {}) + expected = coverage_evidence_bytes(policy_bytes, binding_bytes, + declaration_digest=_digest(declaration), scope_policy_digest=_digest(scope), + baseline_digest=baseline_digest) + if any(active.get(key) != value for key, value in expected.items()): + return CoverageCheck("drift", ("coverage_enrollment_changed",), baseline_digest, None, {}) + try: + current = _scan(declaration, scope, inventory=inventory, + policy_bytes=policy_bytes, binding_bytes=binding_bytes, + enforce_mount_ids=False, postgresql_context=postgresql_context) + except CoverageError: + return CoverageCheck("incomplete", ("inventory_unavailable",), baseline_digest, None, {}) + candidate_digest = _digest(current) + if current["structural_digest"] != baseline.get("structural_digest"): + return CoverageCheck("drift", tuple(sorted(set(current["reasons"]) | {"structural_inventory_changed"})), + baseline_digest, candidate_digest, current["counts"]) + if current["reasons"]: + return CoverageCheck("incomplete", tuple(current["reasons"]), baseline_digest, + candidate_digest, current["counts"]) + roots = tuple((root["id"], Path(root["path"]), root["source_id"], root["adapter"]) + for root in declaration["roots"]) + exclusions = tuple(Path(item["path"]) for item in declaration["exclusions"]) + approved_sqlite = tuple({**item, "baseline_digest": baseline_digest} + for item in current["approved_sqlite"]) + sqlite_capture_exclusions = tuple(Path(path) for path in current["sqlite_capture_exclusions"]) + approved_postgresql = tuple({**item, "baseline_digest": baseline_digest} + for item in current.get("approved_postgresql", ())) + postgresql_capture_exclusions = tuple(Path(path) for path in + current.get("postgresql_capture_exclusions", ())) + if declaration["schema_version"] == 2: + exclusions = tuple(sorted(set(exclusions) | set(postgresql_capture_exclusions), key=str)) + return CoverageCheck("complete", (), baseline_digest, candidate_digest, + current["counts"], roots, exclusions, approved_sqlite, + sqlite_capture_exclusions, + approved_postgresql=approved_postgresql, + postgresql_capture_exclusions=postgresql_capture_exclusions) + + +def evaluate_recovery_evidence(evidence_path: Path | None, *, baseline_digest: str, + snapshot_id: str, source_machine_digest: str) -> dict[str, object]: + """Treat isolated fixture drills as emulation, never whole-machine qualification.""" + for value, label in ((baseline_digest, "baseline"), (snapshot_id, "snapshot"), + (source_machine_digest, "machine")): + _sha(value, label) + if evidence_path is None: + return {"recovery_status": "untested", "reasons": ["recovery_evidence_missing"], + "next_action": "Run an isolated replacement recovery drill"} + try: + record = strict_json(_read_private(Path(evidence_path)).decode("utf-8")) + except (ConfigurationError, UnicodeError): + raise CoverageError("Invalid private recovery evidence") from None + required = {"schema_version", "environment", "baseline_digest", "snapshot_id", + "replacement_machine_digest", "independent_escrow", "checks"} + if not isinstance(record, dict) or set(record) != required or record.get("schema_version") != 1: + raise CoverageError("Invalid recovery evidence schema") + checks = record["checks"] + expected_checks = {"metadata", "secrets", "users", "reinstall", "app_data", "startup"} + if not isinstance(checks, dict) or set(checks) != expected_checks or any(type(value) is not bool for value in checks.values()): + raise CoverageError("Recovery evidence checks are incomplete") + if record["baseline_digest"] != baseline_digest or record["snapshot_id"] != snapshot_id: + raise CoverageError("Recovery evidence differs from exact baseline or snapshot") + replacement = _sha(record["replacement_machine_digest"], "replacement machine") + if record["environment"] != "isolated-local-fixture": + return {"recovery_status": "untested", "reasons": ["independent_qualification_not_verified"], + "next_action": "Verify signed independent replacement evidence"} + reasons = [name + "_unverified" for name, passed in checks.items() if not passed] + if record["independent_escrow"] is not True: + reasons.append("independent_escrow_unverified") + if replacement == source_machine_digest: + reasons.append("replacement_identity_not_distinct") + return {"recovery_status": "emulated" if not reasons else "untested", "reasons": sorted(reasons), + "baseline_digest": baseline_digest, "snapshot_id": snapshot_id, + "next_action": "Obtain signed independent replacement evidence"} diff --git a/bbackup/models.py b/bbackup/models.py index 860e04e..3fb345c 100644 --- a/bbackup/models.py +++ b/bbackup/models.py @@ -76,11 +76,35 @@ class Repository: def parse(cls, raw): raw = _object(raw, {"name", "role"}, {"name"}) role = raw.get("role", "local") - if role not in ("local", "replica"): - raise ConfigurationError("Repository role must be local or replica") + if role not in ("local", "replica", "remote-primary"): + raise ConfigurationError("Repository role must be local, remote-primary, or replica") return cls(_name(raw["name"]), role) +@dataclass(frozen=True) +class ResourceBudget: + """Portable remote-primary staging ceiling and observed free-space reserve.""" + + staging_max_bytes: int + minimum_free_bytes: int + cache_max_bytes: int = 0 + schema_version: int = 1 + + @classmethod + def parse(cls, raw): + raw = _object(raw, set(cls.__dataclass_fields__), + {"schema_version", "staging_max_bytes", "minimum_free_bytes", "cache_max_bytes"}) + if type(raw["schema_version"]) is not int or raw["schema_version"] != 1: + raise ConfigurationError("Unsupported resource budget version") + ceiling = _positive(raw["staging_max_bytes"]) + reserve = _positive(raw["minimum_free_bytes"]) + if type(raw["cache_max_bytes"]) is not int or raw["cache_max_bytes"] != 0: + raise ConfigurationError("Remote-primary cache must be disabled") + if reserve >= ceiling: + raise ConfigurationError("Resource reserve must be below staging capacity") + return cls(ceiling, reserve) + + @dataclass(frozen=True) class Source: name: str @@ -191,6 +215,8 @@ class Job: overdue_hours: int = 26 timeout_seconds: int = 3600 retention: Retention = field(default_factory=Retention) + resource_budget: ResourceBudget | None = None + coverage_policy: dict | None = None @classmethod def parse(cls, raw): @@ -214,6 +240,14 @@ def parse(cls, raw): raise ConfigurationError( "Schedule must be a single nonempty systemd calendar expression" ) + budget = ResourceBudget.parse(raw["resource_budget"]) if "resource_budget" in raw else None + coverage_policy = None + if "coverage_policy" in raw: + from .linux_coverage import validate_scope_policy + try: + coverage_policy = validate_scope_policy(raw["coverage_policy"]) + except (ValueError, RuntimeError) as error: + raise ConfigurationError(str(error)) from None return cls( _name(raw["name"]), _name(raw["repository"]), @@ -223,6 +257,8 @@ def parse(cls, raw): _positive(raw.get("overdue_hours", 26)), _positive(raw.get("timeout_seconds", 3600)), Retention.parse(raw.get("retention", {})), + budget, + coverage_policy, ) @@ -260,11 +296,15 @@ def parse(cls, raw): repositories = {repo.name: repo for repo in config.repositories} sources = {source.name for source in config.sources} for job in config.jobs: - if ( - job.repository not in repositories - or repositories[job.repository].role != "local" - ): - raise ConfigurationError("Jobs require a declared local repository") + if job.coverage_policy is not None and version != 3: + raise ConfigurationError("Coverage scope requires portable policy version 3") + if (job.repository not in repositories + or repositories[job.repository].role not in ("local", "remote-primary")): + raise ConfigurationError("Jobs require a declared primary repository") + if (repositories[job.repository].role == "remote-primary") != (job.resource_budget is not None): + raise ConfigurationError("Remote-primary jobs require a resource budget; local jobs cannot use one") + if repositories[job.repository].role == "remote-primary" and job.replicas: + raise ConfigurationError("Remote-primary jobs cannot dispatch unbudgeted replica copies") if not set(job.sources) <= sources: raise ConfigurationError("Job references an undeclared source") if any( @@ -439,6 +479,8 @@ class HostBindings: docker_bindings: dict[str, DockerBinding] = field(default_factory=dict) source_bindings: dict[str, SourceBinding] = field(default_factory=dict) schema_version: int = 2 + staging_dir: Path | None = None + coverage_declarations: dict[str, Path] = field(default_factory=dict) @classmethod def load(cls, path: Path, configuration: Configuration | None = None): if path.stat().st_mode & 0o077: @@ -453,7 +495,7 @@ def load(cls, path: Path, configuration: Configuration | None = None): def parse(cls, raw, configuration: Configuration | None = None): raw = _object( raw, - {"schema_version", "state_dir", "repositories", "database_bindings", "docker_bindings", "source_bindings"}, + {"schema_version", "state_dir", "repositories", "database_bindings", "docker_bindings", "source_bindings", "staging_dir", "coverage_declarations"}, {"schema_version", "state_dir", "repositories"}, ) version = raw["schema_version"] @@ -474,6 +516,33 @@ def parse(cls, raw, configuration: Configuration | None = None): _name(key): RepositoryBinding.parse(value) for key, value in raw["repositories"].items() } + staging_dir = None + if "staging_dir" in raw: + location = raw["staging_dir"] + if not isinstance(location, str) or not Path(location).is_absolute() or "\0" in location: + raise ConfigurationError("Staging directory must be an absolute private path") + staging_dir = Path(location) + if configuration is not None: + needs_staging = any(r.role == "remote-primary" for r in configuration.repositories) + if needs_staging != (staging_dir is not None): + raise ConfigurationError("Remote-primary bindings require a private staging directory") + raw_coverage = raw.get("coverage_declarations", {}) + if version != 3 and raw_coverage: + raise ConfigurationError("Coverage declarations require version 3 bindings") + if not isinstance(raw_coverage, dict): + raise ConfigurationError("Coverage declarations must be a private job mapping") + coverage_declarations = {} + for name, location in raw_coverage.items(): + name = _name(name) + if (not isinstance(location, str) or not Path(location).is_absolute() + or ".." in Path(location).parts or any(c in location for c in "\0\r\n") + or str(Path(location)) != location): + raise ConfigurationError("Coverage declaration must be an absolute private path") + coverage_declarations[name] = Path(location) + if configuration is not None: + covered = {job.name for job in configuration.jobs if job.coverage_policy is not None} + if set(coverage_declarations) != covered: + raise ConfigurationError("Coverage declarations must match opt-in jobs") locations = [value.repository for value in repositories.values()] if len(set(locations)) != len(locations): raise ConfigurationError("Multiple names for one repository are forbidden") @@ -503,7 +572,7 @@ def parse(cls, raw, configuration: Configuration | None = None): for source in configuration.sources } return cls(Path(state), repositories, database_bindings, docker_bindings, - source_bindings, version) + source_bindings, version, staging_dir, coverage_declarations) @property def sensitive_paths(self) -> tuple[Path, ...]: diff --git a/bbackup/operations.py b/bbackup/operations.py index 3105c8e..f91a877 100644 --- a/bbackup/operations.py +++ b/bbackup/operations.py @@ -234,9 +234,32 @@ def _initialize(self) -> None: exit_code INTEGER, FOREIGN KEY(operation_id) REFERENCES operations(operation_id) ); + CREATE TABLE IF NOT EXISTS capture_artifact_markers ( + operation_id TEXT PRIMARY KEY, + repository_id TEXT NOT NULL, + schema_version INTEGER NOT NULL + ); """ ) + def mark_capture_artifacts(self, repository_id: str, operation_id: str) -> None: + """Persist that a new remote capture requires its private artifact record.""" + + if not re.fullmatch(r"[a-f0-9]{64}", repository_id) or not _SAFE_TOKEN.fullmatch(operation_id): + raise ValueError("Invalid capture artifact marker identity") + with self._connect() as connection: + connection.execute( + "INSERT INTO capture_artifact_markers VALUES (?,?,1)", + (operation_id, repository_id), + ) + + def has_capture_artifact_marker(self, repository_id: str, operation_id: str) -> bool: + with self._connect() as connection: + return connection.execute( + "SELECT 1 FROM capture_artifact_markers WHERE repository_id=? AND operation_id=?", + (repository_id, operation_id), + ).fetchone() is not None + def _lock(self, repository_id: str) -> int: if not re.fullmatch(r"[a-f0-9]{64}", repository_id): raise ValueError("repository_id must be created with repository_key()") @@ -399,6 +422,16 @@ def reconcile( finally: self._unlock(lock_fd) + def inspect(self, repository_id: str, operation_id: str) -> tuple[str, OperationStatus] | None: + """Read an exact operation label and status without changing its lifecycle.""" + + with self._connect() as connection: + row = connection.execute( + "SELECT label, status FROM operations WHERE repository_id=? AND operation_id=?", + (repository_id, operation_id), + ).fetchone() + return (row["label"], OperationStatus(row["status"])) if row else None + def events( self, operation_id: str | None = None, @@ -619,11 +652,16 @@ def run( plan: OperationPlan, *, on_stdout: Callable[[bytes], None] | None = None, + on_stdout_chunk: Callable[[bytes], None] | None = None, on_stderr: Callable[[bytes], None] | None = None, + on_tick: Callable[[], None] | None = None, cancel_event: Event | None = None, ) -> OperationRun: """Execute a plan and return bounded output tails and an explicit code.""" + if on_stdout is not None and on_stdout_chunk is not None: + raise ValueError("Raw stdout sink cannot share line callbacks") + started_at = _now() started = time.monotonic() stdout_tail = _TailBuffer(self.max_buffer_bytes) @@ -691,6 +729,8 @@ def run( callback_error: BaseException | None = None try: while selector.get_map() or process.poll() is None: + if on_tick is not None: + on_tick() if cancel_event is not None and cancel_event.is_set(): status = OperationStatus.CANCELLED self._terminate_group(process) @@ -711,9 +751,13 @@ def run( emitters[fd].feed(b"", final=True) stream.close() continue - tails[fd].append(data) + raw_stdout = stream_names[fd] == "stdout" and on_stdout_chunk is not None + if raw_stdout: + on_stdout_chunk(data) + else: + tails[fd].append(data) + emitters[fd].feed(data) counters[stream_names[fd]] += len(data) - emitters[fd].feed(data) except BaseException as exc: callback_error = exc self._terminate_group(process) @@ -738,9 +782,13 @@ def run( if not data: break drain_budget -= len(data) - tails[fd].append(data) + raw_stdout = stream_names[fd] == "stdout" and on_stdout_chunk is not None + if raw_stdout: + on_stdout_chunk(data) + else: + tails[fd].append(data) + emitters[fd].feed(data) counters[stream_names[fd]] += len(data) - emitters[fd].feed(data) emitters[fd].feed(b"", final=True) except BlockingIOError: pass diff --git a/bbackup/postgresql_coverage.py b/bbackup/postgresql_coverage.py new file mode 100644 index 0000000..fa45157 --- /dev/null +++ b/bbackup/postgresql_coverage.py @@ -0,0 +1,892 @@ +"""Bounded, machine-local PostgreSQL coverage inspection. + +The module inspects one PostgreSQL cluster through the same native operation +runner used by database capture. It records structural digests only; server +settings, credentials, and command output stay in memory. +""" +from __future__ import annotations + +from dataclasses import dataclass +import configparser +import hashlib +import json +import os +from pathlib import Path +import re +import stat +from typing import Any, Mapping + +from .models import DatabaseBinding, Source +from .operations import OperationPlan, OperationRunner, OperationStatus + + +_MAX_OUTPUT = 1024 * 1024 +_MAX_CONNECTION_FILE = 1024 * 1024 +_MAX_RECOVERY_MATERIAL = 64 * 1024 * 1024 +_MAX_DATABASES = 4096 +_MAX_ROLES = 65536 +_MAX_CONFIG_ITEMS = 100000 +_SYSTEM_ID = re.compile(r"^Database system identifier:\s*([0-9]+)\s*$", re.MULTILINE) +_PG_VERSION = re.compile(r"PostgreSQL\)\s+([0-9]+)(?:\.([0-9]+))?") +_DATA_VERSION = re.compile(r"([0-9]+)(?:\.([0-9]+))?\Z") +_INCLUDE = re.compile(r"^\s*(include|include_if_exists|include_dir)\s*(?:=\s*)?(['\"])(.*?)\2\s*(?:#.*)?$", re.I) + + +class PostgreSQLCoverageError(RuntimeError): + """Sanitized PostgreSQL coverage inspection failure.""" + + def __init__(self, reason: str): + self.reason = reason + super().__init__(reason) + + +@dataclass(frozen=True) +class PostgreSQLCoverageContext: + """Resolved native PostgreSQL inputs for a coverage scan. + + ``sources`` is keyed by source name and ``database_bindings`` by the exact + private connection selector. The supplied runner is the normal bounded + ``OperationRunner``; command output is additionally capped here. + """ + + sources: Mapping[str, Source] + database_bindings: Mapping[str, DatabaseBinding] + runner: OperationRunner + cancel: Any = None + timeout_seconds: float = 15.0 + quiet_administrative_window: bool = False + + def __post_init__(self): + if (not isinstance(self.sources, Mapping) + or not isinstance(self.database_bindings, Mapping) + or not hasattr(self.runner, "run") + or isinstance(self.timeout_seconds, bool) + or not isinstance(self.timeout_seconds, (int, float)) + or not 0 < self.timeout_seconds <= 60): + raise ValueError("Invalid PostgreSQL inspection context") + if type(self.quiet_administrative_window) is not bool: + raise ValueError("Invalid PostgreSQL consistency window") + + +def _canonical(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=True).encode("utf-8") + + +def _digest(value: Any) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _within(path: Path, parent: Path) -> bool: + return path == parent or parent in path.parents + + +def _no_symlink_ancestors(path: Path) -> bool: + current = path + while True: + try: + if stat.S_ISLNK(current.lstat().st_mode): + return False + except OSError: + return False + if current.parent == current: + return True + current = current.parent + + +def _private_bytes(path: Path, limit: int) -> bytes: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() + or info.st_mode & 0o077): + raise PostgreSQLCoverageError("postgresql_private_material_unsafe") + data = stream.read(limit + 1) + except PostgreSQLCoverageError: + raise + except OSError: + raise PostgreSQLCoverageError("postgresql_private_material_unavailable") from None + if len(data) > limit: + raise PostgreSQLCoverageError("postgresql_private_material_too_large") + return data + + +def _private_file_digest(path: Path, limit: int) -> str: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() + or info.st_mode & 0o077): + raise PostgreSQLCoverageError("postgresql_private_material_unsafe") + digest = hashlib.sha256() + total = 0 + while True: + chunk = stream.read(min(65536, limit + 1 - total)) + if not chunk: + break + total += len(chunk) + if total > limit: + raise PostgreSQLCoverageError("postgresql_private_material_too_large") + digest.update(chunk) + except PostgreSQLCoverageError: + raise + except OSError: + raise PostgreSQLCoverageError("postgresql_private_material_unavailable") from None + return digest.hexdigest() + + +def _read_cluster_config(data_directory: Path, config_files: list[str], + auth_files: set[Path]) -> str: + """Hash the active config tree while rejecting external files and links.""" + visited: set[Path] = set() + file_digests: dict[Path, str] = {} + auto_config = data_directory / "postgresql.auto.conf" + pending = [Path(item) for item in config_files] + if auto_config.exists() or auto_config.is_symlink(): + pending.append(auto_config) + while pending: + path = pending.pop() + if not path.is_absolute(): + raise PostgreSQLCoverageError("postgresql_external_configuration") + path = Path(os.path.normpath(path)) + if not _within(path, data_directory) or not _no_symlink_ancestors(path): + raise PostgreSQLCoverageError("postgresql_external_configuration") + if path in visited: + continue + visited.add(path) + try: + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise PostgreSQLCoverageError("postgresql_configuration_unsupported") + content = _private_or_server_file(path, 4 * 1024 * 1024) + except PostgreSQLCoverageError: + raise + except OSError: + raise PostgreSQLCoverageError("postgresql_configuration_unavailable") from None + try: + lines = content.decode("utf-8").splitlines() + except UnicodeError: + raise PostgreSQLCoverageError("postgresql_configuration_malformed") from None + file_digests[path] = hashlib.sha256(content).hexdigest() + for line in lines: + stripped = line.lstrip() + if not stripped or stripped.startswith("#"): + continue + if path in auth_files and ( + re.match(r"^(?:include|include_if_exists|include_dir)\s", stripped, re.I) + or "@" in stripped): + raise PostgreSQLCoverageError("postgresql_hba_dependencies_unsupported") + match = _INCLUDE.fullmatch(line) + if match: + kind, _, value = match.groups() + candidate = Path(value) + if not candidate.is_absolute(): + candidate = path.parent / candidate + candidate = Path(os.path.normpath(candidate)) + if not _within(candidate, data_directory): + raise PostgreSQLCoverageError("postgresql_external_configuration") + if kind.lower() == "include_dir": + if not _no_symlink_ancestors(candidate): + raise PostgreSQLCoverageError("postgresql_external_configuration") + try: + children = sorted(candidate.glob("*.conf")) + except OSError: + raise PostgreSQLCoverageError("postgresql_configuration_unavailable") from None + if len(children) > 4096: + raise PostgreSQLCoverageError("postgresql_configuration_limit") + pending.extend(children) + else: + if not candidate.exists() and kind.lower() == "include_if_exists": + continue + pending.append(candidate) + continue + if re.match(r"^\s*include(?:_if_exists|_dir)?(?:\s|=)", line, re.I): + raise PostgreSQLCoverageError("postgresql_configuration_malformed") + if len(visited) > 4096: + raise PostgreSQLCoverageError("postgresql_configuration_limit") + records = [] + for path in sorted(visited, key=str): + records.append([str(path.relative_to(data_directory)), + file_digests[path]]) + return _digest(records) + + +def _private_or_server_file(path: Path, limit: int) -> bytes: + """Read a server-owned configuration file without persisting its contents.""" + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode): + raise PostgreSQLCoverageError("postgresql_configuration_unsupported") + content = stream.read(limit + 1) + except PostgreSQLCoverageError: + raise + except OSError: + raise PostgreSQLCoverageError("postgresql_configuration_unavailable") from None + if len(content) > limit: + raise PostgreSQLCoverageError("postgresql_configuration_limit") + return content + + +def _validate_connection_file(path: Path, service_name: str) -> tuple[str, dict[str, str]]: + content = _private_bytes(path, _MAX_CONNECTION_FILE) + parser = configparser.ConfigParser(interpolation=None, strict=True, delimiters=("="), + comment_prefixes=("#", ";"), inline_comment_prefixes=None) + try: + parser.read_string(content.decode("utf-8")) + if service_name not in parser: + raise ValueError() + service = {key.lower(): value.strip() for key, value in parser[service_name].items()} + except (UnicodeError, configparser.Error, ValueError): + raise PostgreSQLCoverageError("postgresql_connection_file_malformed") from None + if "hostaddr" in service or "include" in service or "include_dir" in service: + raise PostgreSQLCoverageError("postgresql_remote_endpoint_unsupported") + host = service.get("host") + if host and (not host.startswith("/") or "," in host or "\0" in host or "\n" in host): + raise PostgreSQLCoverageError("postgresql_remote_endpoint_unsupported") + if host and not _no_symlink_ancestors(Path(host)): + raise PostgreSQLCoverageError("postgresql_connection_socket_unsafe") + return hashlib.sha256(content).hexdigest(), service + + +def _environment(binding: DatabaseBinding) -> dict[str, str]: + if binding.service_file is None or binding.service_name is None: + raise PostgreSQLCoverageError("postgresql_binding_missing") + env = {"PATH": os.environ.get("PATH", os.defpath), "LC_ALL": "C", + "PGSERVICEFILE": str(binding.service_file), + "PGPASSFILE": str(binding.password_file) if binding.password_file else "/dev/null"} + return env + + +def _run_bounded(context: PostgreSQLCoverageContext, repository: str, + argv: list[str], *, env: dict[str, str]) -> bytes: + output = bytearray() + + def collect(chunk: bytes) -> None: + if not isinstance(chunk, bytes) or len(output) + len(chunk) > _MAX_OUTPUT: + raise PostgreSQLCoverageError("postgresql_inspection_output_limit") + output.extend(chunk) + + plan = OperationPlan.create(repository, "pg-cov", argv, env=env, + timeout_seconds=float(context.timeout_seconds)) + try: + result = context.runner.run(plan, on_stdout_chunk=collect, + cancel_event=context.cancel) + except PostgreSQLCoverageError: + raise + except Exception: + raise PostgreSQLCoverageError("postgresql_inspection_failed") from None + if (result.status is not OperationStatus.SUCCEEDED or result.exit_code != 0 + or result.stdout_bytes != len(output) or len(output) > _MAX_OUTPUT): + raise PostgreSQLCoverageError("postgresql_inspection_incomplete") + return bytes(output) + + +def _parse_local_control(context: PostgreSQLCoverageContext, cluster: dict[str, Any], + env: dict[str, str]) -> tuple[str, str, str]: + data_directory = Path(cluster["data_directory"]) + try: + data_version_text = _private_or_server_file(data_directory / "PG_VERSION", 32).decode("ascii").strip() + except (PostgreSQLCoverageError, UnicodeError): + raise PostgreSQLCoverageError("postgresql_local_identity_malformed") from None + data_version = _DATA_VERSION.fullmatch(data_version_text) + if data_version is None: + raise PostgreSQLCoverageError("postgresql_local_identity_malformed") + data_major = data_version.group(1) + if data_major == "9" and data_version.group(2) is not None: + data_major += "." + data_version.group(2) + version_output = _run_bounded(context, cluster["connection"], + ["pg_controldata", "--version"], env=env) + try: + version_text = version_output.decode("utf-8").strip() + except UnicodeError: + raise PostgreSQLCoverageError("postgresql_local_identity_malformed") from None + version_match = _PG_VERSION.search(version_text) + if version_match is None: + raise PostgreSQLCoverageError("postgresql_local_identity_malformed") + local_major = version_match.group(1) + control_output = _run_bounded(context, cluster["connection"], + ["pg_controldata", str(data_directory)], env=env) + try: + control_text = control_output.decode("utf-8") + except UnicodeError: + raise PostgreSQLCoverageError("postgresql_local_identity_malformed") from None + system_id = _SYSTEM_ID.search(control_text) + if system_id is None: + raise PostgreSQLCoverageError("postgresql_local_identity_malformed") + return system_id.group(1), local_major, data_major + + +def _parse_sql_snapshot(context: PostgreSQLCoverageContext, database: str, + connection_selector: str, + binding: DatabaseBinding, env: dict[str, str]) -> dict[str, Any]: + escaped_database = database.replace("\\", "\\\\").replace("'", "\\'") + connection = f"service={binding.service_name} dbname='{escaped_database}'" + query = """SELECT json_build_object( + 'system_identifier', (pg_control_system()).system_identifier::text, + 'current_database', current_database(), + 'server_version_num', current_setting('server_version_num')::integer, + 'data_directory', current_setting('data_directory'), + 'config_file', current_setting('config_file'), + 'hba_file', current_setting('hba_file'), + 'ident_file', current_setting('ident_file'), + 'in_recovery', pg_is_in_recovery(), + 'databases', COALESCE((SELECT json_agg(json_build_object( + 'name', d.datname, 'oid', d.oid::bigint, 'allow_connections', d.datallowconn, + 'encoding', pg_encoding_to_char(d.encoding), 'collate', d.datcollate, + 'ctype', d.datctype, 'locale_provider', COALESCE(to_jsonb(d)->>'datlocprovider','c'), + 'owner', owner.rolname, 'acl', d.datacl, 'connection_limit', d.datconnlimit, + 'is_template', d.datistemplate + ) ORDER BY d.datname) FROM pg_database AS d + JOIN pg_roles AS owner ON owner.oid=d.datdba), '[]'::json), + 'roles', COALESCE((SELECT json_agg(json_build_object( + 'name', r.rolname, 'superuser', r.rolsuper, 'inherit', r.rolinherit, + 'create_role', r.rolcreaterole, 'create_database', r.rolcreatedb, + 'can_login', r.rolcanlogin, 'replication', r.rolreplication, + 'bypass_rls', r.rolbypassrls, 'connection_limit', r.rolconnlimit, + 'valid_until', r.rolvaliduntil::text, 'password', r.rolpassword + ) ORDER BY r.rolname) FROM pg_authid AS r), '[]'::json), + 'memberships', COALESCE((SELECT json_agg(json_build_object( + 'role', role_name.rolname, 'member', member_name.rolname, + 'admin', m.admin_option, + 'inherit', COALESCE((to_jsonb(m)->>'inherit_option')::boolean,true), + 'set', COALESCE((to_jsonb(m)->>'set_option')::boolean,true) + ) ORDER BY role_name.rolname, member_name.rolname) + FROM pg_auth_members AS m + JOIN pg_roles AS role_name ON role_name.oid=m.roleid + JOIN pg_roles AS member_name ON member_name.oid=m.member), '[]'::json), + 'extensions', COALESCE((SELECT json_agg(json_build_object( + 'name', e.extname, 'version', e.extversion) ORDER BY e.extname) + FROM pg_extension AS e), '[]'::json), + 'languages', COALESCE((SELECT json_agg(lanname ORDER BY lanname) + FROM pg_language WHERE lanname NOT IN ('internal','c','sql','plpgsql')), '[]'::json), + 'foreign_tables', (SELECT count(*) FROM pg_foreign_table), + 'foreign_servers', (SELECT count(*) FROM pg_foreign_server), + 'subscriptions', (SELECT count(*) FROM pg_subscription), + 'replication_slots', (SELECT count(*) FROM pg_replication_slots), + 'tablespaces', COALESCE((SELECT json_agg(json_build_object( + 'name', t.spcname, 'location', pg_tablespace_location(t.oid)) ORDER BY t.spcname) + FROM pg_tablespace AS t), '[]'::json), + 'settings', COALESCE((SELECT json_agg(json_build_array(name,setting,source,context) + ORDER BY name) FROM pg_settings), '[]'::json), + 'role_settings', COALESCE((SELECT json_agg(json_build_object( + 'database', COALESCE(d.datname,''), 'role', COALESCE(r.rolname,''), + 'settings', s.setconfig) ORDER BY d.datname NULLS FIRST, r.rolname NULLS FIRST) + FROM pg_db_role_setting AS s + LEFT JOIN pg_database AS d ON d.oid=s.setdatabase + LEFT JOIN pg_roles AS r ON r.oid=s.setrole), '[]'::json) + )::text;""" + output = _run_bounded(context, connection_selector, + ["psql", "--no-psqlrc", "--no-password", "--quiet", + "--tuples-only", "--no-align", "--set=ON_ERROR_STOP=1", + f"--dbname={connection}", "--command=" + query], env=env) + try: + if not output or len(output) > _MAX_OUTPUT: + raise ValueError() + value = json.loads(output.decode("utf-8").strip()) + except (UnicodeError, ValueError, TypeError): + raise PostgreSQLCoverageError("postgresql_sql_inventory_malformed") from None + required = {"system_identifier", "current_database", "server_version_num", "data_directory", "config_file", + "hba_file", "ident_file", "in_recovery", "databases", "roles", "memberships", + "extensions", "languages", "foreign_tables", "foreign_servers", "subscriptions", + "replication_slots", "tablespaces", "settings", "role_settings"} + if not isinstance(value, dict) or set(value) != required: + raise PostgreSQLCoverageError("postgresql_sql_inventory_malformed") + if (not isinstance(value["system_identifier"], str) + or not re.fullmatch(r"[0-9]{1,32}", value["system_identifier"]) + or not isinstance(value["current_database"], str) + or not value["current_database"] + or type(value["server_version_num"]) is not int + or value["server_version_num"] < 100000 + or type(value["in_recovery"]) is not bool): + raise PostgreSQLCoverageError("postgresql_sql_inventory_malformed") + if any(not isinstance(value[name], str) or not value[name] for name in + ("data_directory", "config_file", "hba_file", "ident_file")): + raise PostgreSQLCoverageError("postgresql_sql_inventory_malformed") + for name in ("databases", "roles", "memberships", "extensions", "languages", + "tablespaces", "settings", "role_settings"): + if not isinstance(value[name], list): + raise PostgreSQLCoverageError("postgresql_sql_inventory_malformed") + if (len(value["databases"]) > _MAX_DATABASES or len(value["roles"]) > _MAX_ROLES + or any(len(value[name]) > _MAX_CONFIG_ITEMS for name in + ("memberships", "extensions", "languages", "tablespaces", "settings", "role_settings"))): + raise PostgreSQLCoverageError("postgresql_sql_inventory_limit") + try: + for role in value["roles"]: + expected = {"name", "superuser", "inherit", "create_role", "create_database", + "can_login", "replication", "bypass_rls", "connection_limit", "valid_until", "password"} + if (not isinstance(role, dict) or set(role) != expected + or not isinstance(role["name"], str) or not role["name"] or len(role["name"]) > 63 + or any(type(role[key]) is not bool for key in + ("superuser", "inherit", "create_role", "create_database", "can_login", + "replication", "bypass_rls")) + or type(role["connection_limit"]) is not int + or (role["valid_until"] is not None and not isinstance(role["valid_until"], str)) + or (role["password"] is not None and not isinstance(role["password"], str))): + raise ValueError() + # The native command output remains in memory. Only a digest of + # each verifier reaches structural evidence or capture metadata. + role["password"] = (hashlib.sha256(role["password"].encode()).hexdigest() + if role["password"] is not None else None) + for membership in value["memberships"]: + if (not isinstance(membership, dict) + or set(membership) != {"role", "member", "admin", "inherit", "set"} + or any(not isinstance(membership[key], str) or not membership[key] + for key in ("role", "member")) + or any(type(membership[key]) is not bool for key in ("admin", "inherit", "set"))): + raise ValueError() + for extension in value["extensions"]: + if (not isinstance(extension, dict) or set(extension) != {"name", "version"} + or any(not isinstance(extension[key], str) or not extension[key] + for key in ("name", "version"))): + raise ValueError() + if any(not isinstance(language, str) or not language for language in value["languages"]): + raise ValueError() + for tablespace in value["tablespaces"]: + if (not isinstance(tablespace, dict) or set(tablespace) != {"name", "location"} + or not isinstance(tablespace["name"], str) or not tablespace["name"] + or tablespace["location"] is not None and not isinstance(tablespace["location"], str)): + raise ValueError() + for setting in value["settings"]: + if (not isinstance(setting, list) or len(setting) != 4 + or not isinstance(setting[0], str) or not setting[0] + or not isinstance(setting[1], str) + or setting[2] is not None and not isinstance(setting[2], str) + or setting[3] is not None and not isinstance(setting[3], str)): + raise ValueError() + for setting in value["role_settings"]: + if (not isinstance(setting, dict) or set(setting) != {"database", "role", "settings"} + or any(not isinstance(setting[key], str) for key in ("database", "role")) + or not isinstance(setting["settings"], list) + or any(not isinstance(item, str) for item in setting["settings"])): + raise ValueError() + for key in ("foreign_tables", "foreign_servers", "subscriptions", "replication_slots"): + if type(value[key]) is not int or value[key] < 0: + raise ValueError() + except (TypeError, ValueError): + raise PostgreSQLCoverageError("postgresql_sql_inventory_malformed") from None + return value + + +def _database_inventory(snapshot: dict[str, Any]) -> list[dict[str, Any]]: + databases = [] + try: + for item in snapshot["databases"]: + if (not isinstance(item, dict) + or set(item) != {"name", "oid", "allow_connections", "encoding", "collate", + "ctype", "locale_provider", "owner", "acl", "connection_limit", + "is_template"} + or not isinstance(item["name"], str) or not item["name"] or len(item["name"]) > 128 + or type(item["oid"]) is not int or item["oid"] <= 0 + or type(item["allow_connections"]) is not bool + or type(item["is_template"]) is not bool + or type(item["connection_limit"]) is not int + or not isinstance(item["owner"], str) or not item["owner"] + or (item["acl"] is not None and (not isinstance(item["acl"], list) + or any(not isinstance(entry, str) for entry in item["acl"]))) + or any(not isinstance(item[key], str) or not item[key] for key in + ("encoding", "collate", "ctype", "locale_provider"))): + raise ValueError() + databases.append({key: item[key] for key in + ("name", "oid", "allow_connections", "encoding", "collate", "ctype", + "locale_provider", "owner", "acl", "connection_limit", "is_template")}) + except (TypeError, ValueError): + raise PostgreSQLCoverageError("postgresql_sql_inventory_malformed") from None + if len({item["name"] for item in databases}) != len(databases) or len({item["oid"] for item in databases}) != len(databases): + raise PostgreSQLCoverageError("postgresql_sql_inventory_ambiguous") + return sorted(databases, key=lambda item: item["name"]) + + +def _verify_database_decisions(cluster: dict[str, Any], database_roots: list[dict[str, Any]], + databases: list[dict[str, Any]], context: PostgreSQLCoverageContext, + reasons: set[str]) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: + declarations = {item["name"]: item for item in cluster["databases"]} + observed = {item["name"]: item for item in databases} + if set(declarations) != set(observed): + reasons.add("postgresql_database_accounting_changed") + for name, decision in declarations.items(): + db = observed.get(name) + if db is None or db["oid"] != decision["oid"]: + reasons.add("postgresql_database_identity_changed") + roots_by_id = {item["id"]: item for item in database_roots} + selected: list[dict[str, Any]] = [] + for name, decision in declarations.items(): + db = observed.get(name) + if decision["decision"] == "reconstruct": + continue + root = roots_by_id.get(decision["root_id"]) + if root is None or root["database"] != name: + reasons.add("postgresql_database_selector_mismatch") + continue + source = context.sources.get(root["source_id"]) + binding = context.database_bindings.get(root["connection"]) + if (source is None or source.name != root["source_id"] or source.kind != "postgresql" + or source.connection != root["connection"] or source.database != root["database"] + or binding is None or binding.kind != "postgresql" + or binding.service_name is None or binding.service_file is None): + reasons.add("postgresql_database_selector_mismatch") + continue + if db is not None and not db["allow_connections"]: + reasons.add("postgresql_database_not_connectable") + selected.append({"root_id": root["id"], "source_id": root["source_id"], + "connection": root["connection"], "database": root["database"], + "cluster_id": root["cluster_id"], "database_oid": decision["oid"]}) + declared_capture_ids = {item["root_id"] for item in cluster["databases"] + if item["decision"] == "capture"} + if declared_capture_ids != {item["id"] for item in database_roots}: + reasons.add("postgresql_database_accounting_changed") + return sorted(selected, key=lambda item: item["root_id"]), [ + {"path": str(Path(item["path"])), "sha256": item["sha256"]} + for item in cluster["reconstruction"]["recovery_material"]] + + +def _verify_reconstruction(cluster: dict[str, Any], *, cluster_digest: str, + global_digest: str, role_digest: str, config_digest: str, + reasons: set[str], data_directory: Path, + physical_roots: list[dict[str, Any]], + exclusions: list[dict[str, Any]], + nonconnectable: list[dict[str, Any]]) -> list[dict[str, str]]: + reconstruction = cluster["reconstruction"] + recipe = reconstruction["recipe"] + procedure_path = Path(reconstruction["procedure_path"]) + evidence_path = Path(reconstruction["evidence_path"]) + try: + procedure_digest = _private_file_digest(procedure_path, 4 * 1024 * 1024) + evidence_bytes = _private_bytes(evidence_path, 1024 * 1024) + evidence = json.loads(evidence_bytes.decode("utf-8")) + except (PostgreSQLCoverageError, UnicodeError, ValueError, TypeError): + reasons.add("postgresql_reconstruction_evidence_unverified") + evidence = None + procedure_digest = "" + evidence_bytes = b"" + material = [] + required_artifacts = [ + {"path": str(procedure_path), "sha256": procedure_digest}, + {"path": str(evidence_path), "sha256": hashlib.sha256(evidence_bytes).hexdigest()}, + ] + for item in reconstruction["recovery_material"]: + path = Path(item["path"]) + try: + digest = _private_file_digest(path, _MAX_RECOVERY_MATERIAL) + except PostgreSQLCoverageError: + reasons.add("postgresql_recovery_material_unavailable") + digest = "" + if digest != item["sha256"]: + reasons.add("postgresql_recovery_material_changed") + material.append({"path": str(path), "sha256": digest}) + material.sort(key=lambda item: item["path"]) + required_artifacts.extend(material) + excluded_paths = [Path(item["path"]) for item in exclusions] + for item in required_artifacts: + path = Path(item["path"]) + if (_within(path, data_directory) or not path.is_absolute() + or not _no_symlink_ancestors(path)): + reasons.add("postgresql_recovery_artifact_route_unsafe") + matching_roots = [root for root in physical_roots + if root["adapter"] == "files" + and _within(path, Path(root["path"]))] + if (len(matching_roots) != 1 + or any(_within(path, excluded) for excluded in excluded_paths)): + reasons.add("postgresql_recovery_artifact_route_uncovered") + else: + item["root_id"] = matching_roots[0]["id"] + if hashlib.sha256(evidence_bytes).hexdigest() != reconstruction["evidence_sha256"]: + reasons.add("postgresql_reconstruction_evidence_unverified") + checks = {"procedure_executed": True, "result_verified": True, + "quiet_administrative_window": True, "schema_changes_frozen": True} + expected_keys = {"schema_version", "recipe_id", "recipe_sha256", "environment", + "cluster_inventory_sha256", "global_inventory_sha256", "role_inventory_sha256", + "configuration_sha256", "recovery_material", "checks", + "nonconnectable_databases"} + expected_nonconnectable = sorted(({ + "name": item["name"], "oid": item["oid"], "metadata_sha256": _digest(item), + "checks": {"procedure_executed": True, "result_verified": True}} + for item in nonconnectable), key=lambda item: item["name"]) + if (procedure_digest != recipe["recipe_sha256"] or not isinstance(evidence, dict) + or set(evidence) != expected_keys or evidence.get("schema_version") != 1 + or evidence.get("recipe_id") != recipe["recipe_id"] + or evidence.get("recipe_sha256") != recipe["recipe_sha256"] + or evidence.get("environment") not in ("isolated-local-fixture", "reviewed-local-procedure") + or evidence.get("cluster_inventory_sha256") != cluster_digest + or evidence.get("global_inventory_sha256") != global_digest + or evidence.get("role_inventory_sha256") != role_digest + or evidence.get("configuration_sha256") != config_digest + or evidence.get("nonconnectable_databases") != expected_nonconnectable + or evidence.get("recovery_material") != [ + {"path": item["path"], "sha256": item["sha256"]} for item in material] + or evidence.get("checks") != checks): + reasons.add("postgresql_reconstruction_evidence_stale") + return required_artifacts + + +def inspect_postgresql_coverage(database_roots: list[dict[str, Any]], + clusters: list[dict[str, Any]], + context: PostgreSQLCoverageContext | None, + physical_roots: list[dict[str, Any]] | None = None, + exclusions: list[dict[str, Any]] | None = None) -> dict[str, Any]: + """Inspect the selected local primary cluster and return digest-only facts.""" + if not database_roots and not clusters: + return {"reasons": [], "approved_postgresql": [], + "capture_exclusions": [], "clusters": []} + if context is None: + return {"reasons": ["postgresql_inspection_context_missing"], + "approved_postgresql": [], "capture_exclusions": [], "clusters": []} + reasons: set[str] = set() + if not context.quiet_administrative_window: + reasons.add("postgresql_quiet_administrative_window_required") + physical_roots = physical_roots or [] + exclusions = exclusions or [] + if len(clusters) != 1: + reasons.add("postgresql_single_cluster_required") + declared_sources = {root["source_id"] for root in database_roots} + resolved_sources = {source.name for source in context.sources.values() + if getattr(source, "kind", None) == "postgresql"} + if (declared_sources != resolved_sources + or any(name != source.name for name, source in context.sources.items())): + reasons.add("postgresql_database_selector_mismatch") + approved: list[dict[str, Any]] = [] + capture_exclusions: list[str] = [] + observed_clusters: list[dict[str, Any]] = [] + roots_by_cluster: dict[str, list[dict[str, Any]]] = {} + for root in database_roots: + roots_by_cluster.setdefault(root["cluster_id"], []).append(root) + for cluster in clusters: + cluster_reasons_before = set(reasons) + directory = Path(cluster["data_directory"]) + try: + info = directory.lstat() + if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) + or not _no_symlink_ancestors(directory) + or info.st_uid == 0 or info.st_mode & 0o027): + raise PostgreSQLCoverageError("postgresql_data_directory_unsafe") + except PostgreSQLCoverageError as exc: + reasons.add(exc.reason) + continue + except OSError: + reasons.add("postgresql_data_directory_unavailable") + continue + pg_wal = directory / "pg_wal" + pg_tblspc = directory / "pg_tblspc" + try: + wal_info = pg_wal.lstat() + if (not stat.S_ISDIR(wal_info.st_mode) or stat.S_ISLNK(wal_info.st_mode) + or wal_info.st_dev != info.st_dev): + reasons.add("postgresql_external_wal_unsupported") + with os.scandir(pg_tblspc) as entries: + if any(True for _ in entries): + reasons.add("postgresql_external_tablespace_unsupported") + except OSError: + reasons.add("postgresql_storage_layout_incomplete") + continue + cluster_roots = roots_by_cluster.get(cluster["id"], []) + selected_sources = [context.sources.get(root["source_id"]) for root in cluster_roots] + matching_sources = [source for source in selected_sources + if source is not None and source.connection == cluster["connection"] + and source.kind == "postgresql"] + if not matching_sources: + reasons.add("postgresql_database_selector_mismatch") + continue + # All selected databases in one cluster share a private connection file. + source = matching_sources[0] + binding = context.database_bindings.get(cluster["connection"]) + if binding is None or binding.kind != "postgresql" or binding.service_file is None: + reasons.add("postgresql_binding_missing") + continue + if any(candidate is None or candidate.connection != cluster["connection"] + or candidate.kind != "postgresql" for candidate in selected_sources): + reasons.add("postgresql_database_selector_mismatch") + try: + connection_digest, _service = _validate_connection_file( + binding.service_file, binding.service_name or "") + password_digest = (_private_file_digest(binding.password_file, _MAX_CONNECTION_FILE) + if binding.password_file is not None else hashlib.sha256(b"no-password-file").hexdigest()) + env = _environment(binding) + snapshot = _parse_sql_snapshot(context, source.database, + cluster["connection"], binding, env) + system_id, local_major, data_major = _parse_local_control(context, cluster, env) + except PostgreSQLCoverageError as exc: + reasons.add(exc.reason) + continue + try: + databases = _database_inventory(snapshot) + except PostgreSQLCoverageError as exc: + reasons.add(exc.reason) + continue + snapshots = {source.database: snapshot} + try: + for db in databases: + if db["allow_connections"] and db["name"] not in snapshots: + snapshots[db["name"]] = _parse_sql_snapshot( + context, db["name"], cluster["connection"], binding, env) + except PostgreSQLCoverageError as exc: + reasons.add(exc.reason) + continue + nonconnectable = [db for db in databases if not db["allow_connections"]] + declared_reconstruct = {item["name"] for item in cluster["databases"] + if item["decision"] == "reconstruct"} + if any(db["name"] not in declared_reconstruct for db in nonconnectable): + reasons.add("postgresql_database_not_connectable") + for database_name, database_snapshot in snapshots.items(): + if database_snapshot["current_database"] != database_name: + reasons.add("postgresql_database_selector_mismatch") + if any(database_snapshot[key] != snapshot[key] for key in + ("system_identifier", "server_version_num", "data_directory", "config_file", + "hba_file", "ident_file", "in_recovery", "databases", "roles", "memberships", + "tablespaces", "settings", "role_settings", "replication_slots")): + reasons.add("postgresql_cluster_inventory_inconsistent") + if database_snapshot["extensions"]: + names = {item["name"] for item in database_snapshot["extensions"]} + if names - {"plpgsql"}: + reasons.add("postgresql_extensions_unsupported") + if database_snapshot["languages"]: + reasons.add("postgresql_languages_unsupported") + for key, reason in (("foreign_tables", "postgresql_foreign_tables_unsupported"), + ("foreign_servers", "postgresql_foreign_servers_unsupported"), + ("subscriptions", "postgresql_subscriptions_unsupported")): + if database_snapshot[key]: + reasons.add(reason) + sql_system_id = snapshot["system_identifier"] + sql_major = str(snapshot["server_version_num"] // 10000) + if system_id != sql_system_id: + reasons.add("postgresql_sql_local_identity_mismatch") + if local_major != sql_major or data_major != sql_major: + reasons.add("postgresql_major_version_mismatch") + if snapshot["in_recovery"]: + reasons.add("postgresql_standby_unsupported") + if str(Path(os.path.normpath(snapshot["data_directory"]))) != str(directory): + reasons.add("postgresql_data_directory_namespace_mismatch") + if any(item["locale_provider"] != "c" for item in databases): + reasons.add("postgresql_non_libc_locale_unsupported") + for name in ("config_file", "hba_file", "ident_file"): + value = snapshot[name] + if not isinstance(value, str) or not value: + reasons.add("postgresql_external_configuration") + continue + path = Path(os.path.normpath(value)) + if not path.is_absolute() or not _within(path, directory): + reasons.add("postgresql_external_configuration") + try: + config_digest = _read_cluster_config(directory, [snapshot["config_file"], + snapshot["hba_file"], snapshot["ident_file"]], + {Path(os.path.normpath(snapshot["hba_file"])), + Path(os.path.normpath(snapshot["ident_file"]))}) + except PostgreSQLCoverageError as exc: + reasons.add(exc.reason) + continue + settings = {item[0]: item[1] for item in snapshot["settings"]} + for name, allowed in {"ssl": {"off"}, "archive_mode": {"off"}, + "shared_preload_libraries": {""}, + "session_preload_libraries": {""}, + "local_preload_libraries": {""}, + "restore_command": {""}, + # PostgreSQL reports this inactive default while + # archive_mode is off, including in version 17. + "archive_command": ({"", "(disabled)"} + if settings.get("archive_mode") == "off" else {""}), + "primary_conninfo": {""}, + "primary_slot_name": {""}}.items(): + if name not in settings or settings[name] not in allowed: + reasons.add("postgresql_configuration_dependencies_unsupported") + for key in ("replication_slots",): + if type(snapshot[key]) is not int or snapshot[key] < 0: + reasons.add("postgresql_sql_inventory_malformed") + elif snapshot[key]: + reasons.add("postgresql_replication_slots_unsupported") + external_tablespaces = [item for item in snapshot["tablespaces"] + if not isinstance(item, dict) or item.get("name") not in ("pg_default", "pg_global") + or item.get("location") not in ("", None)] + if external_tablespaces: + reasons.add("postgresql_external_tablespace_unsupported") + role_data = {"roles": snapshot["roles"], "memberships": snapshot["memberships"]} + role_digest = _digest(role_data) + config_runtime_digest = _digest({"settings": snapshot["settings"], + "role_settings": snapshot["role_settings"], + "files": config_digest}) + database_digests = {name: _digest({"extensions": item["extensions"], + "languages": item["languages"], + "foreign_tables": item["foreign_tables"], + "foreign_servers": item["foreign_servers"], + "subscriptions": item["subscriptions"]}) + for name, item in snapshots.items()} + cluster_digest = _digest({"databases": databases, + "database_inventories": database_digests, + "tablespaces": snapshot["tablespaces"], + "replication_slots": snapshot["replication_slots"]}) + global_digest = _digest({"roles": role_digest, "configuration": config_runtime_digest}) + selected, _ = _verify_database_decisions(cluster, + roots_by_cluster.get(cluster["id"], []), databases, context, reasons) + artifacts = _verify_reconstruction(cluster, cluster_digest=cluster_digest, + global_digest=global_digest, role_digest=role_digest, + config_digest=config_runtime_digest, reasons=reasons, + data_directory=directory, physical_roots=physical_roots, exclusions=exclusions, + nonconnectable=nonconnectable) + for item in artifacts: + try: + if _private_file_digest(Path(item["path"]), _MAX_RECOVERY_MATERIAL) != item["sha256"]: + reasons.add("postgresql_recovery_material_changed_during_inspection") + except PostgreSQLCoverageError: + reasons.add("postgresql_recovery_material_unavailable") + try: + connection_digest_after, _ = _validate_connection_file( + binding.service_file, binding.service_name or "") + password_digest_after = (_private_file_digest(binding.password_file, _MAX_CONNECTION_FILE) + if binding.password_file is not None + else hashlib.sha256(b"no-password-file").hexdigest()) + final_info = directory.lstat() + original_identity = (info.st_dev, info.st_ino, info.st_uid, info.st_gid, + stat.S_IMODE(info.st_mode)) + final_identity = (final_info.st_dev, final_info.st_ino, final_info.st_uid, + final_info.st_gid, stat.S_IMODE(final_info.st_mode)) + if original_identity != final_identity: + reasons.add("postgresql_data_directory_changed_during_inspection") + if (connection_digest != connection_digest_after + or password_digest != password_digest_after): + reasons.add("postgresql_connection_material_changed_during_inspection") + except PostgreSQLCoverageError as exc: + reasons.add(exc.reason) + if (len(clusters) == 1 and system_id == sql_system_id and local_major == sql_major + and data_major == sql_major + and reasons == cluster_reasons_before): + capture_exclusions.append(str(directory)) + observed_clusters.append({"id": cluster["id"], "connection": cluster["connection"], + "data_directory": str(directory), + "data_directory_identity": {"device": info.st_dev, "inode": info.st_ino, + "uid": info.st_uid, "gid": info.st_gid, + "mode": stat.S_IMODE(info.st_mode)}, + "system_identifier_sha256": hashlib.sha256(system_id.encode("ascii")).hexdigest(), + "major_version": sql_major, "cluster_inventory_sha256": cluster_digest, + "global_inventory_sha256": global_digest, + "role_inventory_sha256": role_digest, + "configuration_sha256": config_runtime_digest, + "connection_material_sha256": _digest({"service_file": connection_digest, + "password_file": password_digest}), + "recipe_sha256": cluster["reconstruction"]["recipe"]["recipe_sha256"], + "recovery_artifacts": artifacts, "databases": databases, + "database_inventory_sha256": database_digests}) + approved.extend({**item, "system_identifier_sha256": hashlib.sha256(system_id.encode("ascii")).hexdigest(), + "data_directory": str(directory), "major_version": sql_major, + "data_directory_identity": {"device": info.st_dev, "inode": info.st_ino, + "uid": info.st_uid, "gid": info.st_gid, + "mode": stat.S_IMODE(info.st_mode)}, + "cluster_inventory_sha256": cluster_digest, + "global_inventory_sha256": global_digest, + "role_inventory_sha256": role_digest, + "configuration_sha256": config_runtime_digest, + "connection_material_sha256": _digest({"service_file": connection_digest, + "password_file": password_digest}), + "database_inventory_sha256": database_digests[item["database"]], + "recipe_sha256": cluster["reconstruction"]["recipe"]["recipe_sha256"], + "recovery_artifacts": artifacts} + for item in selected) + reasons.update({"postgresql_context_cancelled"} if context.cancel is not None + and getattr(context.cancel, "is_set", lambda: False)() else set()) + return {"reasons": sorted(reasons), "approved_postgresql": approved, + "capture_exclusions": sorted(set(capture_exclusions)), + "clusters": observed_clusters} + + +__all__ = ["PostgreSQLCoverageContext", "PostgreSQLCoverageError", + "inspect_postgresql_coverage"] diff --git a/bbackup/production_cli.py b/bbackup/production_cli.py index a63d3c9..10070e7 100644 --- a/bbackup/production_cli.py +++ b/bbackup/production_cli.py @@ -15,12 +15,13 @@ from .b2 import B2Error from .cloud import CloudError from .recovery import RecoveryError +from .linux_coverage import CoverageError from .cli_metadata import PRODUCTION_COMMANDS, production_schema from .configuration_workspace import ConfigurationWorkspace from .enrollment import parse_pair, read_pair, verify_bytes from .models import ConfigurationError, strict_json from .operations import OperationError, RepositoryConflictError, UncertainOperationError -from .service import ReplicaJobError, ServiceError +from .service import ReplicaJobError, ResourceBudgetExceeded, ServiceError from .production_operations import _operation_lease, execute_operation from .progress import observe_progress @@ -171,7 +172,13 @@ def callback(**kwargs): "errors": [{"code": "replica_incomplete", "message": str(exc)}]}, sort_keys=True)) ctx.exit(3) - except (ServiceError, OperationError, CaptureError, CloudError, B2Error, RecoveryError, tarfile.TarError, sqlite3.Error, OSError, ValueError, KeyError): + except ResourceBudgetExceeded as exc: + click.echo(json.dumps({"schema_version": 2, "command": command, "success": False, + "data": exc.result, + "errors": [{"code": "resource_budget_exceeded", "message": str(exc)}]}, + sort_keys=True)) + ctx.exit(3) + except (ServiceError, OperationError, CaptureError, CloudError, B2Error, RecoveryError, CoverageError, tarfile.TarError, sqlite3.Error, OSError, ValueError, KeyError): _emit(command, error={"code": "operation_failed", "message": "Operation failed; inspect sanitized run state before retrying"}) ctx.exit(3) params = [click.Option(["--input-json"], type=str), click.Option(["--schema"], is_flag=True), diff --git a/bbackup/production_operations.py b/bbackup/production_operations.py index 37f8001..7af26ed 100644 --- a/bbackup/production_operations.py +++ b/bbackup/production_operations.py @@ -4,19 +4,25 @@ from dataclasses import asdict from contextlib import contextmanager from pathlib import Path +import re import shutil from typing import Any +from urllib.parse import urlsplit -from .b2 import inspect_b2 +from .b2 import B2VersionLockReader, inspect_b2 from .cli_metadata import PRODUCTION_COMMANDS, production_schema -from .cloud import S3CloudAdapter +from .cloud import S3CloudAdapter, _normalize_endpoint_url from .configuration_workspace import ConfigurationWorkspace, read_document -from .enrollment import parse_pair, policy_lease, read_pair, verify_bytes -from .models import ConfigurationError -from .operations import ReconciliationOutcome, repository_key -from .recovery import (create_checked_kit, open_kit, verify_checkpoint, +from .enrollment import (_read_regular, machine_identity, parse_pair, policy_lease, + read_pair, verify_bytes) +from .linux_coverage import (check_coverage, discover_coverage, + evaluate_recovery_evidence, review_coverage) +from .postgresql_coverage import PostgreSQLCoverageContext +from .models import ConfigurationError, strict_json +from .operations import OperationRunner, ReconciliationOutcome, repository_key +from .recovery import (accept_recovery_checkpoint, create_checked_kit, open_kit, verify_checkpoint, verify_reconstructed_repository) -from .scheduling import install_user_units, job_units +from .scheduling import install_user_units, job_units, system_job_units from .service import BackupService, ServiceError from .service_status import operational_status @@ -66,7 +72,8 @@ def _operation_lease(config_path, bindings_path): verify_bytes(current_bindings.state_dir, policy_bytes, binding_bytes) if read_pair(config_path, bindings_path) != (policy_bytes, binding_bytes): raise ConfigurationError("Configuration changed during enrollment validation; review and retry") - yield BackupService(current_config, current_bindings) + yield BackupService(current_config, current_bindings, + policy_bytes=policy_bytes, binding_bytes=binding_bytes) def execute_operation(command: str, values: dict[str, Any], *, service: BackupService | None = None, @@ -106,6 +113,17 @@ def execute_operation(command: str, values: dict[str, Any], *, service: BackupSe return workspace.save(read_document(Path(values["document_file"])), values["revision"]) if command.startswith("recovery ") and command != "recovery checkpoint": return _recovery_dispatch(service, command, values, cancel) + if command == "coverage review": + # Review owns an exclusive policy lease; do not nest it inside the + # ordinary shared operation lease and deadlock the same process. + if config_path is None or bindings_path is None: + raise ConfigurationError("Coverage review requires configuration and bindings paths") + policy_bytes, binding_bytes = read_pair(config_path, bindings_path) + configuration, bindings = parse_pair(policy_bytes, binding_bytes) + if configuration.schema_version != 3: + raise ConfigurationError("Coverage review requires version-three enrollment") + return _coverage_operation(configuration, bindings, policy_bytes, binding_bytes, + command, values, cancel=cancel) if command == "doctor" and config_path is not None and bindings_path is not None: policy_bytes, binding_bytes = read_pair(config_path, bindings_path) configuration, bindings = parse_pair(policy_bytes, binding_bytes) @@ -130,6 +148,14 @@ def execute_operation(command: str, values: dict[str, Any], *, service: BackupSe def _service_operation(service, command, values, config_path, bindings_path, cancel): + if command.startswith("coverage "): + return _coverage_operation(service.config, service.bindings, + service.policy_bytes, service.binding_bytes, command, values, + cancel=cancel) + if command.startswith("cloud-retention "): + return _cloud_retention_operation(service, command, values, cancel) + if command.startswith("cloud-maintenance "): + return _cloud_maintenance_operation(service, command, values, cancel) if command == "doctor": return {"restic_available": shutil.which("restic") is not None, "jobs": len(service.config.jobs), "production_qualified": False, @@ -137,7 +163,8 @@ def _service_operation(service, command, values, config_path, bindings_path, can if command == "jobs list": return [asdict(job) for job in service.config.jobs] if command == "jobs run": - return service.run_job(values["name"], cancel=cancel) + return service.run_job(values["name"], cancel=cancel, + quiet_administrative_window_confirmed=values["quiet_administrative_window_confirmed"]) if command == "repositories list": return [asdict(repo) for repo in service.config.repositories] if command == "repositories init": @@ -195,16 +222,24 @@ def _service_operation(service, command, values, config_path, bindings_path, can outcome = ReconciliationOutcome(values["outcome"]) except ValueError: raise ConfigurationError("Invalid reconciliation outcome") from None - service.ledger.reconcile(repository_key(service._identity(values["repository"])), values["operation_id"], outcome) - return {"operation_id": values["operation_id"], "reviewed_outcome": outcome.value, "replayed": False} + return service.reconcile_run(values["repository"], values["operation_id"], outcome, + quiescent_window_confirmed=values["quiescent_window_confirmed"], + cancel=cancel) + if command == "runs retained-exports": + return service.retained_exports(values["repository"], limit=values["limit"], + offset=values["offset"]) + if command == "runs discard-exports": + return service.discard_exports(values["repository"], values["operation_id"], + values["artifact_record_sha256"]) if command in ("runs list", "runs stream"): events = service.ledger.events(**values) return [asdict(event) for event in events] - if command == "service render": + if command in ("service render", "service render-system"): if config_path is None or bindings_path is None: raise ConfigurationError("Rendering services requires configuration paths") - return job_units(service._job(values["name"]), executable=Path(values["executable"]), - config=config_path.absolute(), bindings=bindings_path.absolute()) + render = system_job_units if command == "service render-system" else job_units + return render(service._job(values["name"]), executable=Path(values["executable"]), + config=config_path.absolute(), bindings=bindings_path.absolute()) if command == "service install": if config_path is None or bindings_path is None: raise ConfigurationError("Installing services requires configuration paths") @@ -214,6 +249,150 @@ def _service_operation(service, command, values, config_path, bindings_path, can raise ServiceError("Unsupported command") +def _coverage_operation(configuration, bindings, policy_bytes, binding_bytes, command, values, + *, cancel=None): + job = next((item for item in configuration.jobs if item.name == values["name"]), None) + if job is None or job.coverage_policy is None or job.name not in bindings.coverage_declarations: + raise ConfigurationError("Job requires portable coverage policy and private declaration") + arguments = {"state": bindings.state_dir, "policy_bytes": policy_bytes, + "binding_bytes": binding_bytes} + pg_sources = {source.name: bindings.source_bindings[source.name].resolve(source) + for source in configuration.sources if source.name in job.sources + and source.kind == "postgresql"} + if pg_sources: + arguments["postgresql_context"] = PostgreSQLCoverageContext( + pg_sources, bindings.database_bindings, OperationRunner(), cancel=cancel, + quiet_administrative_window=values.get("quiet_administrative_window_confirmed", False)) + declaration = bindings.coverage_declarations[job.name] + if command == "coverage discover": + return discover_coverage(declaration, job.coverage_policy, **arguments) + if command == "coverage review": + return review_coverage(declaration, job.coverage_policy, values["candidate_digest"], **arguments) + coverage = check_coverage(declaration, job.coverage_policy, **arguments) + if command == "coverage check": + return coverage.public_result() + if not coverage.capture_ready or coverage.baseline_digest is None: + raise ConfigurationError("Recovery evidence requires the exact current reviewed coverage baseline") + return evaluate_recovery_evidence( + Path(values["evidence_file"]) if "evidence_file" in values else None, + baseline_digest=coverage.baseline_digest, snapshot_id=values["snapshot"], + source_machine_digest=machine_identity()[0]) + + +def _private_json(path): + try: + return strict_json(_read_regular(Path(path), private=True).decode("utf-8")) + except UnicodeError: + raise ConfigurationError("Invalid private JSON input") from None + + +def _cloud_retention_operation(service, command, values, cancel): + """Bind central cloud authority to the enrolled repository and its ledger.""" + adapter, lease_id = _central_cloud_adapter(service, values) + if command == "cloud-retention catalog-enroll": + return adapter.initialize_central_retention_catalog(values["policy_file"], values["grant_file"]) + if command == "cloud-retention reconcile": + return adapter.reconcile_central_retention( + values["plan_file"], ledger=service.ledger, expected_repository_lease_id=lease_id) + directories = _central_record_directories(values["records_file"]) + if command == "cloud-retention plan": + return adapter.plan_central_retention( + values["policy_file"], values["grant_file"], directories, + quiescent_window_confirmed=values["quiescent_window_confirmed"], + repository_lease_id=lease_id) + return adapter.apply_central_retention( + values["plan_id"], values["plan_sha256"], values["policy_file"], values["grant_file"], + directories, quiescent_window_confirmed=values["quiescent_window_confirmed"], + ledger=service.ledger, cancel_event=cancel, expected_repository_lease_id=lease_id) + + +def _central_cloud_adapter(service, values, *, maintenance=False): + """Freeze one inspection/deletion pair for native and S3 authority.""" + policy = _private_json(values["policy_file"]) + if not isinstance(policy, dict): + raise ConfigurationError("Invalid central retention policy") + endpoint = _normalize_endpoint_url(policy.get("endpoint_url")) + bucket, prefix = policy.get("bucket"), policy.get("prefix") + if (not isinstance(bucket, str) or not bucket or "/" in bucket + or not isinstance(prefix, str) or not prefix or prefix.startswith("/")): + raise ConfigurationError("Invalid central retention target") + identity = service._identity(values["repository"]) + expected = f"s3:{endpoint}/{bucket}/{prefix.rstrip('/')}" + if identity.rstrip("/") != expected: + raise ConfigurationError("Central retention target differs from enrolled repository") + native = policy.get("lock_metadata") is not None + freeze_pair = native or policy.get("schema_version") == 2 or (policy.get("schema_version") == 3 and maintenance) + if freeze_pair: + parsed = urlsplit(endpoint) + match = re.fullmatch(r"s3\.([a-z0-9-]+)\.backblazeb2\.com", parsed.netloc) + if parsed.scheme != "https" or match is None or parsed.path not in ("", "/") or parsed.query or parsed.fragment: + raise ConfigurationError("Native B2 retention requires a canonical B2 S3 endpoint") + lock_metadata = policy.get("lock_metadata") + if native and ( + not isinstance(lock_metadata, dict) or set(lock_metadata) != {"provider", "bucket_id"} + or lock_metadata.get("provider") != "b2-native" + or not isinstance(lock_metadata.get("bucket_id"), str) + or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", lock_metadata["bucket_id"]) + ): + raise ConfigurationError("Native B2 lock target is invalid") + try: + import boto3 + from botocore.config import Config + + credentials = boto3.Session().get_credentials() + frozen = credentials.get_frozen_credentials() if credentials is not None else None + if frozen is None or not frozen.access_key or not frozen.secret_key or frozen.token: + raise ValueError() + pair = (frozen.access_key, frozen.secret_key) + reader = (B2VersionLockReader( + bucket, lock_metadata["bucket_id"], prefix, endpoint, + access_key_id=pair[0], secret_access_key=pair[1], + ) if native else None) + client = boto3.client( + "s3", endpoint_url=endpoint, region_name=match.group(1), + aws_access_key_id=pair[0], aws_secret_access_key=pair[1], + config=Config(connect_timeout=10, read_timeout=30, + retries={"total_max_attempts": 5, "mode": "standard"}, + signature_version="s3v4"), + ) + except Exception: + raise ConfigurationError("Could not bind native B2 retention authority") from None + adapter = S3CloudAdapter( + bucket, client=client, endpoint_url=endpoint, region_name=match.group(1), + prefix=prefix, native_lock_reader=reader, credential_pair=pair, + ) + else: + adapter = S3CloudAdapter(bucket, endpoint_url=endpoint, prefix=prefix) + return adapter, repository_key(identity) + + +def _central_record_directories(path): + directories = _private_json(path) + if (not isinstance(directories, list) or not directories or len(directories) > 10000 + or any(not isinstance(path, str) or not Path(path).is_absolute() + or any(char in path for char in "\0\r\n") for path in directories) + or len(set(directories)) != len(directories)): + raise ConfigurationError("Expected unique absolute central acceptance directories") + return directories + + +def _cloud_maintenance_operation(service, command, values, cancel): + adapter, lease_id = _central_cloud_adapter(service, values, maintenance=True) + if command == "cloud-maintenance catalog-enroll": + return adapter.initialize_central_retention_catalog( + values["policy_file"], values["grant_file"], + expected_action="maintain_current_repository", + ) + directories = _central_record_directories(values["records_file"]) + if command == "cloud-maintenance records-enroll": + return adapter.enroll_central_maintenance_records( + values["policy_file"], values["grant_file"], directories, + ) + return service.cloud_maintenance( + command, adapter, values, directories, lease_id=lease_id, cancel=cancel, + ) + + def _recovery_dispatch(service, command, values, cancel): if command == "recovery b2-inspect": return inspect_b2(**values, cancel=cancel) @@ -221,6 +400,13 @@ def _recovery_dispatch(service, command, values, cancel): return open_kit(Path(values["kit"]), Path(values["target"]), signer=values["signer"], cancel=cancel) adapter = S3CloudAdapter(values["bucket"], endpoint_url=values["endpoint"], prefix=values.get("prefix", ""), region_name=values.get("region")) + if command == "recovery accept": + return accept_recovery_checkpoint( + adapter, Path(values["kit"]), Path(values["target"]), + source_signer=values["source_signer"], trusted_verifier=values["verifier"], + source_repository=values["repository"], repository_id=values["repository_id"], + snapshot=values["snapshot"], cancel=cancel, + ) if command == "recovery inspect": return asdict(adapter.inspect_lifecycle()) if command == "recovery checkpoint": diff --git a/bbackup/recovery.py b/bbackup/recovery.py index 424cb7d..151f623 100644 --- a/bbackup/recovery.py +++ b/bbackup/recovery.py @@ -1,16 +1,21 @@ """Independent, signed and GnuPG-encrypted recovery kit artifacts.""" from __future__ import annotations +import ctypes +import errno import hashlib +import json import os import stat from contextlib import contextmanager +from datetime import datetime, timezone from pathlib import Path import re import tarfile import tempfile from .operations import OperationPlan, OperationRunner, OperationStatus, repository_key +from .cloud import _COPYRIGHT_URL, _PRIVATE_NOTICE class RecoveryError(RuntimeError): @@ -79,7 +84,10 @@ def _verify_status(result, signer): if fields[1] == "VALIDSIG": if len(fields) < 11: raise RecoveryError("Malformed signature status") - valid.append(fields[2].upper() == signer or fields[-1].upper() == signer) + # The configured identity is the primary key. Accepting a signing + # subkey fingerprint here would let source and verifier be two + # fingerprints from the same OpenPGP certificate. + valid.append(fields[-1].upper() == signer) if valid != [True]: raise RecoveryError("Signature does not match the trusted signer") @@ -282,3 +290,206 @@ def verify_reconstructed_repository(repository: Path, password_file: Path, repos raise RecoveryError("Reconstructed repository identity differs from checkpoint") except (ValueError, AttributeError): raise RecoveryError("Invalid reconstructed repository identity") from None + + +_ACCEPTANCE_FIELDS = { + "type", "schema_version", "manifest_sha256", "source_signer", + "verifier", "source_repository", "repository_id", "snapshot_id", + "captured_at", "expires_at", "verified_at", "files_restored", + "bytes_restored", "recovery_verified", "protection_qualified", + "copyright", "copyright_url", "private_notice", +} +_ID = re.compile(r"[0-9a-f]{64}\Z") +_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z") + + +def _acceptance_time(value): + if not isinstance(value, str) or len(value) > 40: + raise RecoveryError("Invalid recovery acceptance timestamp") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + raise RecoveryError("Invalid recovery acceptance timestamp") from None + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise RecoveryError("Invalid recovery acceptance timestamp") + return parsed.astimezone(timezone.utc) + + +def _acceptance_identity(source_repository, repository_id, snapshot): + if not isinstance(source_repository, str) or not _NAME.fullmatch(source_repository): + raise RecoveryError("Invalid source repository identity") + if not isinstance(repository_id, str) or not _ID.fullmatch(repository_id): + raise RecoveryError("Invalid repository identity") + if not isinstance(snapshot, str) or not _ID.fullmatch(snapshot): + raise RecoveryError("A complete snapshot identity is required") + + +def _acceptance_record(raw, *, verifier, repository_id, snapshot): + try: + record = json.loads(raw, object_pairs_hook=lambda pairs: _unique_pairs(pairs)) + except (ValueError, TypeError, UnicodeDecodeError): + raise RecoveryError("Invalid recovery acceptance record") from None + if not isinstance(record, dict) or set(record) != _ACCEPTANCE_FIELDS: + raise RecoveryError("Invalid recovery acceptance schema") + if record["type"] != "recovery_acceptance" or type(record["schema_version"]) is not int or record["schema_version"] != 1: + raise RecoveryError("Unsupported recovery acceptance schema") + _acceptance_identity(record["source_repository"], record["repository_id"], record["snapshot_id"]) + if record["repository_id"] != repository_id or record["snapshot_id"] != snapshot: + raise RecoveryError("Recovery acceptance identity differs from expected repository") + if not isinstance(record["manifest_sha256"], str) or not _ID.fullmatch(record["manifest_sha256"]): + raise RecoveryError("Invalid recovery acceptance manifest digest") + if _fingerprint(record["verifier"]) != verifier or _fingerprint(record["source_signer"]) == verifier: + raise RecoveryError("Recovery acceptance signer trust differs from central configuration") + if record["recovery_verified"] is not True or record["protection_qualified"] is not False: + raise RecoveryError("Recovery acceptance makes an unsupported claim") + if any(type(record[key]) is not int or record[key] < 0 for key in ("files_restored", "bytes_restored")): + raise RecoveryError("Invalid recovery acceptance restore counts") + captured = _acceptance_time(record["captured_at"]) + verified = _acceptance_time(record["verified_at"]) + expiry = record["expires_at"] + if captured > verified or (expiry is not None and (_acceptance_time(expiry) <= verified or _acceptance_time(expiry) <= captured)): + raise RecoveryError("Recovery acceptance time is outside the checkpoint window") + if (record["copyright"] != f"Copyright © {verified.year} Crux Experts LLC" + or record["copyright_url"] != _COPYRIGHT_URL + or record["private_notice"] != _PRIVATE_NOTICE): + raise RecoveryError("Recovery acceptance ownership metadata is invalid") + return record + + +def _unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate key") + result[key] = value + return result + + +def _publish_acceptance_noreplace(staging: Path, destination: Path): + """Atomically publish a complete directory without replacing even an empty one.""" + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if renameat2 is None: + raise RecoveryError("Atomic exclusive acceptance publication is unavailable") + renameat2.argtypes = (ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint) + renameat2.restype = ctypes.c_int + if renameat2(-100, os.fsencode(staging), -100, os.fsencode(destination), 1) != 0: + if ctypes.get_errno() == errno.EEXIST: + raise RecoveryError("Recovery acceptance destination already exists") + raise RecoveryError("Atomic exclusive acceptance publication failed") + parent_fd = os.open(destination.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(parent_fd) + finally: + os.close(parent_fd) + + +def verify_recovery_acceptance(record_dir: Path, *, trusted_verifier: str, repository_id: str, + snapshot: str, home=None, cancel=None): + """Authenticate a central acceptance against an independently configured verifier key.""" + verifier = _fingerprint(trusted_verifier) + _acceptance_identity("recovered", repository_id, snapshot) + with tempfile.TemporaryDirectory(prefix="bbackup-acceptance-verify-") as temporary: + pinned = Path(temporary) / "acceptance.json" + signature = Path(temporary) / "acceptance.sig" + _copy(Path(record_dir) / "acceptance.json", pinned, limit=16384, cancel=cancel) + _copy(Path(record_dir) / "acceptance.sig", signature, limit=1024**2, cancel=cancel) + result = _gpg(["--status-fd", "1", "--verify", "--", str(signature), str(pinned)], home=home, cancel=cancel) + _verify_status(result, verifier) + return _acceptance_record(pinned.read_bytes(), verifier=verifier, repository_id=repository_id, snapshot=snapshot) + + +def accept_recovery_checkpoint(adapter, kit: Path, record_dir: Path, *, source_signer: str, + trusted_verifier: str, source_repository: str, repository_id: str, + snapshot: str, home=None, cancel=None): + """Centrally sign recovery proof after rebuilding and fully checking exact versions. + + The verifier signing key must be held away from the source host. The caller + supplies trusted source and verifier fingerprints from central configuration; + possession of a valid but unconfigured key is insufficient. + """ + source_signer, verifier = _fingerprint(source_signer), _fingerprint(trusted_verifier) + if source_signer == verifier: + raise RecoveryError("Source and central verifier keys must differ") + _acceptance_identity(source_repository, repository_id, snapshot) + record_dir = Path(record_dir) + if record_dir.exists() or record_dir.is_symlink() or not record_dir.parent.is_dir() or any( + parent.is_symlink() for parent in record_dir.parents + ): + raise RecoveryError("Recovery acceptance destination must be a new local directory") + with tempfile.TemporaryDirectory(prefix="bbackup-central-", dir=record_dir.parent) as temporary: + root = Path(temporary) + opened = root / "opened" + authenticated = open_kit(Path(kit), opened, signer=source_signer, home=home, cancel=cancel) + restored = root / "restored" + restored.mkdir(mode=0o700) + result = adapter.restore_checkpoint(opened / "checkpoint.jsonl", restored, + expected_manifest_sha256=authenticated["manifest_sha256"], + expected_source_repository=source_repository, expected_repository_id=repository_id, + expected_source_snapshot_id=snapshot, cancel_event=cancel) + if result.manifest_sha256 != authenticated["manifest_sha256"]: + raise RecoveryError("Reconstructed manifest digest differs from authenticated kit") + verify_reconstructed_repository(restored, opened / "repository.password", repository_id, snapshot, cancel=cancel) + with _regular(opened / "checkpoint.jsonl", _MAX_ARTIFACT) as stream: + header = stream.readline(1024 * 1024 + 1) + try: + from .cloud import _strict_json + source = _strict_json(header) + except (ValueError, TypeError): + raise RecoveryError("Invalid authenticated checkpoint header") from None + if not isinstance(source, dict) or source.get("type") != "checkpoint": + raise RecoveryError("Invalid authenticated checkpoint header") + verified_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + captured_at, expires_at = source.get("captured_at"), source.get("expires_at") + if _acceptance_time(captured_at) > _acceptance_time(verified_at) or ( + expires_at is not None and _acceptance_time(expires_at) <= _acceptance_time(verified_at) + ): + raise RecoveryError("Checkpoint is future-dated or expired at central verification") + record = { + "type": "recovery_acceptance", "schema_version": 1, + "manifest_sha256": authenticated["manifest_sha256"], + "source_signer": source_signer, "verifier": verifier, + "source_repository": source_repository, "repository_id": repository_id, + "snapshot_id": snapshot, "captured_at": captured_at, + "expires_at": expires_at, "verified_at": verified_at, + "files_restored": result.files_restored, "bytes_restored": result.bytes_restored, + "recovery_verified": True, "protection_qualified": False, + "copyright": f"Copyright © {_acceptance_time(verified_at).year} Crux Experts LLC", + "copyright_url": _COPYRIGHT_URL, "private_notice": _PRIVATE_NOTICE, + } + payload = json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + b"\n" + if len(payload) > 16384: + raise RecoveryError("Recovery acceptance record is too large") + _acceptance_record(payload, verifier=verifier, repository_id=repository_id, snapshot=snapshot) + unsigned = root / "acceptance.json" + with unsigned.open("xb") as stream: + unsigned.chmod(0o600) + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + signature = root / "acceptance.sig" + _gpg(["--local-user", verifier, "--output", str(signature), "--detach-sign", str(unsigned)], home=home, cancel=cancel) + verify_recovery_acceptance(root, trusted_verifier=verifier, repository_id=repository_id, snapshot=snapshot, home=home, cancel=cancel) + # Retention needs the exact version dependencies authenticated by this + # acceptance. Preserve the original signed manifest in private custody. + staging = root / "publish" + staging.mkdir(mode=0o700) + members = ( + (unsigned, "acceptance.json", 16384), + (signature, "acceptance.sig", 1024**2), + (opened / "checkpoint.jsonl", "checkpoint.jsonl", _MAX_ARTIFACT), + (opened / "checkpoint.sig", "checkpoint.sig", 1024**2), + ) + for source_path, name, limit in members: + _copy(source_path, staging / name, limit=limit, cancel=cancel) + with (staging / name).open("rb") as stream: + os.fsync(stream.fileno()) + verify_recovery_acceptance(staging, trusted_verifier=verifier, + repository_id=repository_id, snapshot=snapshot, home=home, cancel=cancel) + directory_fd = os.open(staging, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + _publish_acceptance_noreplace(staging, record_dir) + return record diff --git a/bbackup/resource_limits.py b/bbackup/resource_limits.py new file mode 100644 index 0000000..3fc3822 --- /dev/null +++ b/bbackup/resource_limits.py @@ -0,0 +1,144 @@ +"""Remote-primary staging boundary and observed free-space monitoring. + +The dedicated mount provides the hard capacity ceiling. Free-space reserves +are observations; other writers can consume space between checks. +""" +from __future__ import annotations + +import errno +import fcntl +import os +from pathlib import Path +import stat + +from .models import ResourceBudget + + +class ResourceLimitError(RuntimeError): + """A sanitized capacity preflight or monitored reserve failure.""" + + +def _volume(path: Path) -> tuple[int, int]: + try: + info = os.statvfs(path) + except OSError: + raise ResourceLimitError("Staging or state filesystem is unavailable") from None + return info.f_blocks * info.f_frsize, info.f_bavail * info.f_frsize + + +class ResourceMonitor: + def __init__(self, staging: Path, state: Path, budget: ResourceBudget, *, + read_only_reconcile: bool = False): + self.staging, self.state, self.budget = Path(staging), Path(state), budget + self.read_only_reconcile = read_only_reconcile + self._lock_fd = None + if (not self.staging.is_absolute() or self.staging.resolve() != self.staging + or not os.path.ismount(self.staging)): + raise ResourceLimitError("Remote staging must be a dedicated mounted filesystem") + try: + info = self.staging.lstat() + except OSError: + raise ResourceLimitError("Remote staging is unavailable") from None + if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() + or info.st_mode & 0o077): + raise ResourceLimitError("Remote staging must be an owned private directory") + lock = self.staging / ".bbackup-staging.lock" + try: + descriptor = os.open(lock, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + self._lock_fd = descriptor + lock_info = os.fstat(descriptor) + if (not stat.S_ISREG(lock_info.st_mode) or lock_info.st_uid != os.getuid() + or lock_info.st_mode & 0o077 or lock_info.st_nlink != 1 + or lock_info.st_dev != info.st_dev): + raise ResourceLimitError("Remote staging lock is not an owned private file") + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + if exc.errno in (errno.EACCES, errno.EAGAIN): + raise ResourceLimitError("Remote staging is busy") from None + raise ResourceLimitError("Remote staging lock is unavailable") from None + if not read_only_reconcile and any(path.name != lock.name for path in self.staging.iterdir()): + raise ResourceLimitError("Remote staging must be empty before capture") + if lock.lstat().st_ino != lock_info.st_ino: + raise ResourceLimitError("Remote staging lock changed during acquisition") + capacity, _ = _volume(self.staging) + except OSError: + self.close() + raise ResourceLimitError("Remote staging lock or mount is unavailable") from None + except BaseException: + self.close() + raise + try: + if capacity <= 0 or capacity > budget.staging_max_bytes: + raise ResourceLimitError("Mounted staging capacity exceeds the configured budget") + self.capacity = capacity + self.device = info.st_dev + self.lock_inode = lock_info.st_ino + self.initial_staging_free_bytes = _volume(self.staging)[1] + self.initial_state_free_bytes = _volume(self.state)[1] + self.peak_staging_used_bytes = 0 + self.peak_state_used_bytes = 0 + self.minimum_staging_free_bytes = capacity + self.minimum_state_free_bytes = 2**63 - 1 + if not read_only_reconcile: + self.check() + except BaseException: + self.close() + raise + + def close(self): + if self._lock_fd is not None: + descriptor, self._lock_fd = self._lock_fd, None + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + def check(self, upcoming_bytes: int = 0) -> None: + if type(upcoming_bytes) is not int or upcoming_bytes < 0: + raise ValueError("upcoming_bytes must be nonnegative") + if self._lock_fd is None: + raise ResourceLimitError("Remote staging lease is closed") + lock = self.staging / ".bbackup-staging.lock" + try: + lock_info = lock.lstat() + except OSError: + raise ResourceLimitError("Remote staging lock changed during capture") from None + if (lock_info.st_ino != self.lock_inode or lock_info.st_dev != self.device + or not stat.S_ISREG(lock_info.st_mode)): + raise ResourceLimitError("Remote staging lock changed during capture") + capacity, available = _volume(self.staging) + state_capacity, state_available = _volume(self.state) + try: + device = self.staging.stat().st_dev + except OSError: + raise ResourceLimitError("Staging mount changed during capture") from None + if (capacity != self.capacity or device != self.device or not os.path.ismount(self.staging)): + raise ResourceLimitError("Staging mount changed during capture") + self.peak_staging_used_bytes = max(self.peak_staging_used_bytes, + self.initial_staging_free_bytes - available) + self.peak_state_used_bytes = max(self.peak_state_used_bytes, + self.initial_state_free_bytes - state_available) + self.minimum_staging_free_bytes = min(self.minimum_staging_free_bytes, available) + self.minimum_state_free_bytes = min(self.minimum_state_free_bytes, state_available) + if (available - upcoming_bytes < self.budget.minimum_free_bytes + or state_available < self.budget.minimum_free_bytes): + raise ResourceLimitError("Observed free-space reserve fell below the configured threshold") + + def result(self) -> dict[str, int | bool]: + return { + "staging_capacity_bytes": self.capacity, + "staging_observed_peak_additional_used_bytes": self.peak_staging_used_bytes, + "state_observed_peak_additional_used_bytes": self.peak_state_used_bytes, + "staging_minimum_free_bytes": self.minimum_staging_free_bytes, + "state_minimum_free_bytes": self.minimum_state_free_bytes, + "staging_capacity_bounded": True, + "free_space_reserve_hard_guaranteed": False, + "usage_attributed_to_this_operation": False, + } diff --git a/bbackup/scheduling.py b/bbackup/scheduling.py index c15a782..1eb8527 100644 --- a/bbackup/scheduling.py +++ b/bbackup/scheduling.py @@ -17,12 +17,13 @@ def _argument(path: Path) -> str: return '"' + value.replace('\\', '\\\\').replace('"', '\\"').replace('%', '%%') + '"' -def _service_unit(command: str) -> str: +def _service_unit(command: str, *, private_tmp: bool = True) -> str: return ( "[Unit]\nDescription=bbackup scheduled capture\nWants=network-online.target\n" "After=network-online.target\n\n[Service]\nType=exec\n" f"ExecStart={command}\nUMask=0077\nKillMode=control-group\n" - "TimeoutStopSec=30\nNoNewPrivileges=yes\nPrivateTmp=yes\n" + "TimeoutStopSec=30\nNoNewPrivileges=yes\n" + f"PrivateTmp={'yes' if private_tmp else 'no'}\n" ) @@ -40,13 +41,52 @@ def job_units(job: Job, *, executable: Path, config: Path, bindings: Path) -> di "--bindings", _argument(bindings), "jobs", "run", "--name", job.name)) service_unit = f"bbackup-{job.name}.service" return { - service_unit: _service_unit(command), + # Covered capture must see the same explicitly reviewed filesystem + # namespace as discovery, including any approved temporary paths. + service_unit: _service_unit(command, private_tmp=job.coverage_policy is None), f"bbackup-{job.name}.timer": _timer_unit( job.schedule, service_unit, "bbackup capture schedule" ), } +def _root_owned_path(path: Path, *, executable: bool = False, private: bool = False) -> None: + """Check the privilege boundary before rendering a root system unit.""" + if not path.is_absolute() or ".." in path.parts or any(c in str(path) for c in "\0\r\n"): + raise ValueError("Root unit inputs require canonical absolute paths") + current = Path("/") + for part in path.parts[1:-1]: + current /= part + try: + info = current.lstat() + except OSError: + raise ValueError("Root unit input ancestor is unavailable") from None + if (not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022): + raise ValueError("Root unit input ancestor is not root controlled") + try: + info = path.lstat() + except OSError: + raise ValueError("Root unit input is unavailable") from None + if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022 + or (private and info.st_mode & 0o077) + or (executable and not info.st_mode & 0o111)): + raise ValueError("Root unit input is not root controlled") + + +def system_job_units(job: Job, *, executable: Path, config: Path, bindings: Path) -> dict[str, str]: + """Render a root system service and persistent timer without installing them.""" + if os.geteuid() != 0: + raise ValueError("Root system unit rendering requires root identity") + _root_owned_path(executable, executable=True) + _root_owned_path(config) + _root_owned_path(bindings, private=True) + units = job_units(job, executable=executable, config=config, bindings=bindings) + service = f"bbackup-{job.name}.service" + units[service] = units[service].replace("[Service]\nType=exec\n", + "[Service]\nType=exec\nUser=root\nGroup=root\n") + return units + + def unit_templates( *, executable: Path = Path("/usr/bin/bbackup"), diff --git a/bbackup/service.py b/bbackup/service.py index bb8879e..f6f6fb3 100644 --- a/bbackup/service.py +++ b/bbackup/service.py @@ -12,15 +12,21 @@ import re import shutil import sqlite3 +import stat +import sys import tempfile import uuid +from urllib.parse import urlsplit -from .capture import capture_sources +from .capture import CaptureError, DurableCaptureArtifacts, capture_sources +from .cloud import _retention_json_bytes from .config import SnapshotProfile -from .models import Configuration, HostBindings +from .models import Configuration, HostBindings, ResourceBudget, strict_json from .docker_recovery import DockerRecoveryError, DockerRecoveryUncertainError, DockerRuntime from .snapshot import ResticRunner +from .resource_limits import ResourceLimitError, ResourceMonitor from .progress import stdout_progress_handler +from .enrollment import _read_regular from .operations import ( OperationLedger, OperationPlan, @@ -44,6 +50,21 @@ def __init__(self, result: dict): super().__init__("One or more replica copies failed; review each destination before retrying") +class ResourceBudgetExceeded(ServiceError): + """Sanitized capacity failure with evidence and no credential or source paths.""" + + def __init__(self, result: dict): + self.result = result + super().__init__("Remote-primary resource budget was exceeded or cannot be verified") + + +class _RemoteResticRunner(ResticRunner): + def base_args(self): + args = super().base_args() + args.insert(1, "--no-cache") + return args + + def _snapshot_id(value: str) -> str: if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value): raise ServiceError("A complete snapshot ID is required") @@ -54,13 +75,92 @@ def _overlap(a: Path, b: Path) -> bool: return a == b or a in b.parents or b in a.parents +def _covered_postgresql_refs(entry: dict, *, source: str, + snapshot_paths: list[str]) -> list[dict]: + """Validate the covered export and its file routes before restore dispatch.""" + coverage = entry.get("coverage") + if coverage is None: + return [] + if (not isinstance(snapshot_paths, list) + or any(not isinstance(path, str) for path in snapshot_paths)): + raise ServiceError("Covered PostgreSQL snapshot paths are invalid") + digest = re.compile(r"[0-9a-f]{64}\Z") + if (not isinstance(coverage, dict) or coverage.get("source_id") != source + or not isinstance(coverage.get("connection"), str) + or not coverage["connection"] + or not isinstance(coverage.get("database"), str) + or coverage.get("database") != entry.get("source_database") + or not isinstance(coverage.get("root_id"), str) + or not isinstance(coverage.get("cluster_id"), str) + or not isinstance(coverage.get("major_version"), str) + or not isinstance(coverage.get("data_directory"), str) + or not isinstance(entry.get("export_sha256"), str) + or not digest.fullmatch(entry["export_sha256"])): + raise ServiceError("Covered PostgreSQL export identity is invalid") + for key in ("baseline_digest", "system_identifier_sha256", "cluster_inventory_sha256", + "global_inventory_sha256", "role_inventory_sha256", "configuration_sha256", + "connection_material_sha256", "database_inventory_sha256", "recipe_sha256"): + if not isinstance(coverage.get(key), str) or not digest.fullmatch(coverage[key]): + raise ServiceError("Covered PostgreSQL evidence is invalid") + identity = coverage.get("data_directory_identity") + if (not isinstance(identity, dict) or set(identity) != {"device", "inode", "uid", "gid", "mode"} + or any(type(value) is not int or value < 0 for value in identity.values())): + raise ServiceError("Covered PostgreSQL directory identity is invalid") + roots = coverage.get("recovery_file_roots") + artifacts = coverage.get("recovery_artifacts") + if (not isinstance(roots, list) or not roots or len(roots) > 64 + or not isinstance(artifacts, list) or len(artifacts) < 3 or len(artifacts) > 130): + raise ServiceError("Covered PostgreSQL recovery routes are invalid") + root_paths = {} + for root in roots: + if (not isinstance(root, dict) or set(root) != {"root_id", "path"} + or not isinstance(root["root_id"], str) + or not isinstance(root["path"], str)): + raise ServiceError("Covered PostgreSQL recovery routes are invalid") + path = Path(root["path"]) + if (not path.is_absolute() or ".." in path.parts or root["path"] not in snapshot_paths + or root["root_id"] in root_paths): + raise ServiceError("Covered PostgreSQL recovery route was not captured") + root_paths[root["root_id"]] = path + seen = set() + for item in artifacts: + if (not isinstance(item, dict) or set(item) != {"path", "sha256", "root_id"} + or not isinstance(item["path"], str) + or not isinstance(item["root_id"], str) + or not isinstance(item["sha256"], str) + or not digest.fullmatch(item["sha256"]) or item["path"] in seen): + raise ServiceError("Covered PostgreSQL recovery artifact is invalid") + path = Path(item["path"]) + root = root_paths.get(item["root_id"]) + if (root is None or not path.is_absolute() or ".." in path.parts + or (path != root and root not in path.parents)): + raise ServiceError("Covered PostgreSQL recovery artifact route is invalid") + seen.add(item["path"]) + return artifacts + + class BackupService: - def __init__(self, config: Configuration, bindings: HostBindings): + def __init__(self, config: Configuration, bindings: HostBindings, *, + policy_bytes: bytes | None = None, binding_bytes: bytes | None = None): self.config, self.bindings = config, bindings + self.policy_bytes, self.binding_bytes = policy_bytes, binding_bytes if config.schema_version != bindings.schema_version: raise ServiceError("Policy and private binding versions must match") if set(bindings.repositories) != {r.name for r in config.repositories}: raise ServiceError("Host bindings must match declared repositories") + self.repository_roles = {repository.name: repository.role for repository in config.repositories} + for name, role in self.repository_roles.items(): + if role != "remote-primary": + continue + location = bindings.repositories[name].repository + try: + remote = urlsplit(location[3:]) if location.startswith("s3:") else None + except ValueError: + remote = None + if (remote is None or remote.scheme != "https" or not remote.hostname + or not remote.path.strip("/") or remote.username or remote.password + or remote.query or remote.fragment or bindings.staging_dir is None): + raise ServiceError("Remote-primary requires an HTTPS S3 binding and private staging") if config.schema_version == 3: if set(bindings.source_bindings) != {source.name for source in config.sources}: raise ServiceError("Private source bindings must match portable sources") @@ -105,8 +205,49 @@ def __init__(self, config: Configuration, bindings: HostBindings): table: {row[1] for row in db.execute(f"PRAGMA table_info({table})")} for table in expected } + checkpoint_info = list(db.execute("PRAGMA table_info(checkpoints)")) + legacy_checkpoint_expiry = any( + row[1] == "expires_at" and row[3] == 1 for row in checkpoint_info + ) + if legacy_checkpoint_expiry: + # Rebuild only the exact owning legacy definition. PRAGMA + # table_info omits CHECK, DEFAULT, COLLATE, and inline UNIQUE; + # recreating such a table would silently discard constraints. + legacy_sql = """CREATE TABLE checkpoints ( + repository TEXT NOT NULL, repository_identity TEXT NOT NULL, + snapshot TEXT NOT NULL, operation TEXT PRIMARY KEY, + manifest_sha256 TEXT NOT NULL, captured_at TEXT NOT NULL, + expires_at TEXT NOT NULL + )""" + stored_sql = db.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='checkpoints'" + ).fetchone() + normalized = re.sub(r"\s+", "", stored_sql[0]).upper() if stored_sql and stored_sql[0] else "" + expected_sql = re.sub(r"\s+", "", legacy_sql).upper() + known_definitions = {expected_sql, + expected_sql.replace("CREATETABLE", "CREATETABLEIFNOTEXISTS", 1)} + expected_layout = ( + ("repository", "TEXT", 1, 0), + ("repository_identity", "TEXT", 1, 0), + ("snapshot", "TEXT", 1, 0), + ("operation", "TEXT", 0, 1), + ("manifest_sha256", "TEXT", 1, 0), + ("captured_at", "TEXT", 1, 0), + ("expires_at", "TEXT", 1, 0), + ) + layout = tuple((row[1], row[2].upper(), row[3], row[5]) + for row in checkpoint_info) + additions = db.execute( + "SELECT 1 FROM sqlite_master WHERE tbl_name='checkpoints' " + "AND type IN ('index', 'trigger') AND sql IS NOT NULL" + ).fetchone() + if (normalized not in known_definitions or layout != expected_layout + or additions is not None + or list(db.execute("PRAGMA foreign_key_list(checkpoints)"))): + raise ServiceError("Checkpoint receipt schema has unsupported constraints") if any(existing.values()) and ( any(not columns.issubset(existing[table]) for table, columns in expected.items()) + or legacy_checkpoint_expiry or db.execute( "SELECT 1 FROM sqlite_master WHERE type='trigger' " "AND name='snapshots_required_replicas_immutable'" @@ -156,9 +297,33 @@ def __init__(self, config: Configuration, bindings: HostBindings): repository TEXT NOT NULL, repository_identity TEXT NOT NULL, snapshot TEXT NOT NULL, operation TEXT PRIMARY KEY, manifest_sha256 TEXT NOT NULL, captured_at TEXT NOT NULL, - expires_at TEXT NOT NULL + expires_at TEXT )""" ) + if legacy_checkpoint_expiry: + replacement = "checkpoints_nullable_upgrade" + if db.execute("SELECT 1 FROM sqlite_master WHERE name = ?", (replacement,)).fetchone(): + raise ServiceError("Checkpoint receipt migration name is occupied") + prior_count = db.execute("SELECT count(*) FROM checkpoints").fetchone()[0] + db.execute( + """CREATE TABLE checkpoints_nullable_upgrade ( + repository TEXT NOT NULL, repository_identity TEXT NOT NULL, + snapshot TEXT NOT NULL, operation TEXT PRIMARY KEY, + manifest_sha256 TEXT NOT NULL, captured_at TEXT NOT NULL, + expires_at TEXT + )""" + ) + db.execute( + """INSERT INTO checkpoints_nullable_upgrade + (repository, repository_identity, snapshot, operation, + manifest_sha256, captured_at, expires_at) + SELECT repository, repository_identity, snapshot, operation, + manifest_sha256, captured_at, expires_at FROM checkpoints""" + ) + if db.execute("SELECT count(*) FROM checkpoints_nullable_upgrade").fetchone()[0] != prior_count: + raise ServiceError("Checkpoint receipt migration did not preserve rows") + db.execute("DROP TABLE checkpoints") + db.execute("ALTER TABLE checkpoints_nullable_upgrade RENAME TO checkpoints") db.execute( """CREATE TRIGGER IF NOT EXISTS snapshots_required_replicas_immutable BEFORE UPDATE OF required_replicas ON snapshots @@ -212,9 +377,46 @@ def _engine(self, name): binding = self.bindings.repositories[name] if not binding.password_file.is_file() or binding.password_file.stat().st_mode & 0o077: raise ServiceError("Repository password file must exist and be private") - return ResticRunner(SnapshotProfile(name=name, repository=binding.repository, + runner_type = _RemoteResticRunner if self.repository_roles[name] == "remote-primary" else ResticRunner + return runner_type(SnapshotProfile(name=name, repository=binding.repository, password_file=str(binding.password_file), retry_lock="0s")) + def _resource_monitor(self, name, *, read_only_reconcile=False): + if self.repository_roles[name] != "remote-primary": + return None + budgets = [job.resource_budget for job in self.config.jobs if job.repository == name] + if not budgets or self.bindings.staging_dir is None: + raise ServiceError("Remote-primary requires a configured resource budget and private staging") + # One repository has one physical staging volume; every job must agree on its ceiling. + if any(budget != budgets[0] for budget in budgets): + raise ServiceError("Remote-primary jobs sharing a repository require one resource budget") + try: + if read_only_reconcile: + return ResourceMonitor(self.bindings.staging_dir, self.state, budgets[0], + read_only_reconcile=True) + return ResourceMonitor(self.bindings.staging_dir, self.state, budgets[0]) + except ResourceLimitError as exc: + raise ResourceBudgetExceeded({"resource_budget_exceeded": True, "reason": str(exc), + "repository": name, "local_complete": False, + "cloud_complete": False, "replayed": False}) from None + + def _operation_environment(self, name): + if name not in self.repository_roles: + raise ServiceError("Unknown repository") + if self.repository_roles[name] != "remote-primary": + return {k: v for k, v in os.environ.items() if not k.startswith("RESTIC_")} + if not os.environ.get("AWS_ACCESS_KEY_ID") or not os.environ.get("AWS_SECRET_ACCESS_KEY"): + raise ServiceError("Remote-primary requires explicit provider environment credentials") + staging = self.bindings.staging_dir + assert staging is not None + environment = {"PATH": os.environ.get("PATH", os.defpath), "HOME": str(staging), + "TMPDIR": str(staging)} + for key in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", + "AWS_DEFAULT_REGION", "AWS_REGION", "AWS_CA_BUNDLE", "SSL_CERT_FILE"): + if key in os.environ: + environment[key] = os.environ[key] + return environment + def _identity(self, name): if name not in self.bindings.repositories: raise ServiceError("Unknown repository") @@ -223,6 +425,243 @@ def _identity(self, name): value = str(Path(value).resolve()) return value + def _maintenance_bindings(self, path, adapter): + """Resolve the separate Restic pair from one bounded private declaration.""" + try: + raw = _read_regular(Path(path), private=True) + value = strict_json(raw.decode("utf-8")) + except (OSError, UnicodeError, ValueError): + raise ServiceError("Private maintenance bindings are invalid") from None + required = {"schema_version", "staging_dir", "access_key_id_env", "secret_access_key_env"} + if (not isinstance(value, dict) or set(value) - required - {"credential_profile"} + or not required <= set(value) or value["schema_version"] != 1 + or type(value["schema_version"]) is not int): + raise ServiceError("Private maintenance bindings are invalid") + profile = value.get("credential_profile", "restricted") + if profile not in {"restricted", "management"}: + raise ServiceError("Maintenance credential profile is invalid") + staging_value = value["staging_dir"] + if (not isinstance(staging_value, str) or not Path(staging_value).is_absolute() + or Path(staging_value).resolve() != Path(staging_value)): + raise ServiceError("Maintenance staging directory must be canonical and absolute") + names = (value["access_key_id_env"], value["secret_access_key_env"]) + if (any(not isinstance(name, str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name) + for name in names) or names[0] == names[1]): + raise ServiceError("Maintenance credential environment references are invalid") + pair = tuple(os.environ.get(name) for name in names) + if (not all(pair) or any("\0" in item for item in pair) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", pair[0])): + raise ServiceError("Maintenance Restic credentials are unavailable") + if getattr(adapter, "_credential_pair", None) == pair: + raise ServiceError("Maintenance Restic credentials must differ from adapter credentials") + staging = Path(staging_value) + environment = { + "PATH": os.environ.get("PATH", os.defpath), + "HOME": str(staging), + "TMPDIR": str(staging), + "AWS_ACCESS_KEY_ID": pair[0], + "AWS_SECRET_ACCESS_KEY": pair[1], + } + endpoint_host = urlsplit(getattr(adapter, "endpoint_url", "") or "").hostname or "" + region = re.fullmatch(r"s3\.([a-z0-9-]+)\.backblazeb2\.com", endpoint_host) + if region is None: + raise ServiceError("Maintenance requires a canonical B2 endpoint") + environment["AWS_DEFAULT_REGION"] = region.group(1) + environment["AWS_REGION"] = region.group(1) + for name in ("SSL_CERT_FILE", "AWS_CA_BUNDLE"): + if name in os.environ: + environment[name] = os.environ[name] + return hashlib.sha256(raw).hexdigest(), {"key_id": pair[0], "profile": profile}, staging, environment, pair + + def _maintenance_inspect_b2(self, adapter, identity, environment, pair, profile, cancel): + """Authenticate target and observed capabilities in a credential-limited child.""" + from .b2 import B2Error + + endpoint = getattr(adapter, "endpoint_url", None) + parsed = urlsplit(endpoint or "") + if (parsed.scheme != "https" or not re.fullmatch( + r"s3\.[a-z0-9-]+\.backblazeb2\.com", parsed.netloc + ) or parsed.path not in ("", "/") or parsed.query or parsed.fragment): + raise ServiceError("Maintenance requires a canonical B2 S3 endpoint") + inspection_env = { + "PATH": environment["PATH"], "HOME": environment["HOME"], + "TMPDIR": environment["TMPDIR"], + "B2_APPLICATION_KEY_ID": pair[0], "B2_APPLICATION_KEY": pair[1], + } + argv = (sys.executable, "-m", "bbackup.b2", "--bucket", adapter.bucket, + "--prefix", adapter.prefix, "--key-id-env", "B2_APPLICATION_KEY_ID", + "--key-env", "B2_APPLICATION_KEY") + result = self.runner.run(OperationPlan.create( + identity, "maintenance-b2-inspect", argv, env=inspection_env, + timeout_seconds=120), cancel_event=cancel) + if (result.status is not OperationStatus.SUCCEEDED or result.exit_code != 0 + or result.stdout_bytes != len(result.stdout_tail)): + raise ServiceError("Maintenance B2 credential inspection failed") + try: + inspection = strict_json(result.stdout_tail.decode("utf-8")) + if (not isinstance(inspection, dict) or inspection.get("bucket") != adapter.bucket + or inspection.get("prefix") != adapter.prefix + or inspection.get("s3_endpoint") != endpoint + or inspection.get("bucket_access_verified") is not True + or inspection.get("version_listing_verified") is not True + or inspection.get("write_capability_present") is not True + or inspection.get("read_capability_present") is not True): + raise ValueError() + if profile == "restricted" and inspection.get("restricted_source_profile", {}).get("status") != "eligible_from_observed_metadata": + raise ValueError() + except (ValueError, UnicodeError, TypeError, AttributeError, B2Error): + raise ServiceError("Maintenance B2 key scope or target is unqualified") from None + + def _maintenance_restic(self, identity, password_file, environment, monitor, argv, + *, cancel, timeout, complete_output=False): + base = ("restic", "-r", identity, "--password-file", str(password_file), + "--no-cache", "--retry-lock", "0s") + plan = OperationPlan.create(identity, "maintenance-restic", (*base, *argv), + env=environment, timeout_seconds=timeout) + monitor.check() + result = self.runner.run(plan, on_tick=monitor.check, cancel_event=cancel) + monitor.check() + if result.status is not OperationStatus.SUCCEEDED or result.exit_code != 0: + raise ServiceError("Maintenance Restic inspection or command failed") + if complete_output and result.stdout_bytes != len(result.stdout_tail): + raise ServiceError("Maintenance Restic metadata exceeded bounded capture") + return result + + def _maintenance_observe(self, identity, password_file, environment, monitor, + *, cancel, timeout, no_lock, check): + prefix = ("--no-lock",) if no_lock else () + def query(*args, complete=True): + return self._maintenance_restic( + identity, password_file, environment, monitor, (*prefix, *args), + cancel=cancel, timeout=timeout, complete_output=complete, + ) + if no_lock and query("list", "locks").stdout_tail.strip() not in {b"", b"[]"}: + raise ServiceError("Maintenance observed an active Restic lock") + try: + config = strict_json(query("cat", "config").stdout_tail.decode("utf-8")) + snapshots = strict_json(query("snapshots", "--json").stdout_tail.decode("utf-8")) + repository_id = config["id"] + ids = sorted(item["id"] for item in snapshots) + if (not re.fullmatch(r"[0-9a-f]{64}", repository_id) or len(ids) != len(set(ids)) + or any(not re.fullmatch(r"[0-9a-f]{64}", item) for item in ids)): + raise ValueError() + except (ValueError, UnicodeError, KeyError, TypeError): + raise ServiceError("Maintenance Restic metadata is invalid or incomplete") from None + if check: + query("check", complete=False) + if no_lock and query("list", "locks").stdout_tail.strip() not in {b"", b"[]"}: + raise ServiceError("Maintenance observed a new Restic lock") + evidence = {"config_repository_id": repository_id, "visible_snapshot_ids": ids} + evidence["snapshot_inventory_sha256"] = hashlib.sha256(_retention_json_bytes(evidence)).hexdigest() + return evidence + + def cloud_maintenance(self, command, adapter, values, directories, *, lease_id, cancel=None): + """Run one central maintenance operation using the existing catalog and ledger.""" + from .resource_limits import ResourceMonitor + + repository = values["repository"] + identity = self._identity(repository) + binding = self.bindings.repositories[repository] + if not binding.password_file.is_file() or binding.password_file.stat().st_mode & 0o077: + raise ServiceError("Maintenance repository password file must be private") + if values["quiescent_window_confirmed"] is not True: + raise ServiceError("Maintenance requires a confirmed quiet window") + digest, credential, staging, environment, pair = self._maintenance_bindings( + values["maintenance_bindings_file"], adapter + ) + with adapter.maintenance_session( + values["policy_file"], values["grant_file"], directories, + quiescent_window_confirmed=True, repository_lease_id=lease_id, + reconcile_only=command == "cloud-maintenance reconcile", + ) as session: + budget = ResourceBudget.parse(dict(session.selection.resources["resource_budget"])) + timeout = session.selection.resources["timeout_seconds"] + try: + monitor = ResourceMonitor(staging, self.state, budget, + read_only_reconcile=command != "cloud-maintenance apply") + except ResourceLimitError: + raise ServiceError("Maintenance staging mount or reserve is unqualified") from None + with monitor: + self._maintenance_inspect_b2(adapter, identity, environment, pair, credential["profile"], cancel) + if command == "cloud-maintenance plan": + observed = self._maintenance_observe( + identity, binding.password_file, environment, monitor, + cancel=cancel, timeout=timeout, no_lock=True, check=False) + return session.plan(observed, bindings_sha256=digest, + credential_identity=credential) + plan_id, plan_sha = values["plan_id"], values["plan_sha256"] + if command == "cloud-maintenance reconcile": + plan = session.reconciliation_plan(plan_id, plan_sha, + bindings_sha256=digest, credential_identity=credential) + observed = self._maintenance_observe( + identity, binding.password_file, environment, monitor, + cancel=cancel, timeout=timeout, no_lock=True, check=True) + outcome = session.finish_reconciliation(plan, { + "schema_version": 1, "repository_health": "healthy", **observed}) + ledger_state = self.ledger.inspect(lease_id, plan_id) + if ledger_state is not None and ledger_state[1] in { + OperationStatus.RUNNING, OperationStatus.UNCERTAIN, + }: + self.ledger.reconcile(lease_id, plan_id, + ReconciliationOutcome.SUCCEEDED if outcome["outcome"] == "succeeded" + else ReconciliationOutcome.ABANDONED) + return outcome + observed = self._maintenance_observe( + identity, binding.password_file, environment, monitor, + cancel=cancel, timeout=timeout, no_lock=False, check=False) + plan = session.validate_plan(plan_id, plan_sha, observed, + bindings_sha256=digest, credential_identity=credential) + ledger_plan = OperationPlan(plan_id, lease_id, "cloud-maintenance", + ("restic", "check"), destructive=True) + with self.ledger.begin(ledger_plan) as lease: + mutation_started = False + try: + def phase(name, command_args=None, evidence=None): + nonlocal mutation_started + session.record_phase(plan, name, "intent") + try: + result = None + if command_args is not None: + if name in {"forget", "prune"}: + mutation_started = True + result = self._maintenance_restic( + identity, binding.password_file, environment, monitor, + command_args, cancel=cancel, timeout=timeout, + complete_output=name == "prune") + value = evidence(result) if callable(evidence) else evidence + session.record_phase(plan, name, "completed", evidence=value) + except BaseException: + session.record_phase(plan, name, + "uncertain" if name in {"forget", "prune"} else "failed") + raise + before = self._maintenance_observe( + identity, binding.password_file, environment, monitor, + cancel=cancel, timeout=timeout, no_lock=False, check=True) + phase("check_before", evidence={"repository_health": "healthy", **before}) + phase("forget", ("forget", "--", *plan["target_snapshot_ids"]) + if plan["target_snapshot_ids"] else None, + {"forgotten_snapshot_ids": plan["target_snapshot_ids"]}) + after_forget = self._maintenance_observe( + identity, binding.password_file, environment, monitor, + cancel=cancel, timeout=timeout, no_lock=False, check=False) + phase("verify_snapshots", evidence=after_forget) + phase("prune", ("prune", "--max-repack-size", "0"), + lambda result: {"result": "completed", "evidence_sha256": + hashlib.sha256(result.stdout_tail + b"\0" + result.stderr_tail).hexdigest()}) + after_prune = self._maintenance_observe( + identity, binding.password_file, environment, monitor, + cancel=cancel, timeout=timeout, no_lock=False, check=True) + phase("check_after", evidence={"repository_health": "healthy", **after_prune}) + phase("protected_versions", evidence=session.verify_protected_versions()) + outcome = session.finish_reconciliation(plan, { + "schema_version": 1, "repository_health": "healthy", **after_prune}) + lease.finish(OperationStatus.SUCCEEDED, 0) + return outcome + except BaseException: + if not mutation_started: + lease.finish(OperationStatus.FAILED, None) + raise + def _execute( self, name, @@ -235,14 +674,30 @@ def _execute( mutating=False, finalize: Callable[[OperationPlan, OperationRun], object] | None = None, on_stdout=None, + resource_monitor=None, after_run: Callable[[], None] | None = None, timeout_seconds: int = 3600, ): if cancel is not None and cancel.is_set(): raise ServiceError("Operation cancelled before dispatch") # Restic configuration cannot be redirected by inherited RESTIC_* options. - env = {k: v for k, v in os.environ.items() if not k.startswith("RESTIC_")} repositories = sorted(set((name, *other)), key=self._identity) + remote_participants = [repository for repository in repositories + if self.repository_roles[repository] == "remote-primary"] + env = self._operation_environment(remote_participants[0] if remote_participants else name) + if remote_participants and argv and argv[0] == "restic" and "--no-cache" not in argv: + argv = (argv[0], "--no-cache", *argv[1:]) + elif argv and argv[0] == "restic": + has_explicit_cache = any( + argument == "--no-cache" or argument == "--cache-dir" + or argument.startswith("--cache-dir=") + for argument in argv[1:] + ) + has_cache_environment = any( + env.get(variable) for variable in ("HOME", "XDG_CACHE_HOME", "RESTIC_CACHE_DIR") + ) + if not has_explicit_cache and not has_cache_environment: + argv = (argv[0], "--no-cache", *argv[1:]) plans = {} for repository in repositories: identity = self._identity(repository) @@ -255,6 +710,9 @@ def _execute( timeout_seconds=timeout_seconds, ) with ExitStack() as stack: + if remote_participants and resource_monitor is None: + resource_monitor = self._resource_monitor(remote_participants[0]) + stack.callback(resource_monitor.close) leases = [] try: for plan in plans.values(): @@ -270,20 +728,31 @@ def _execute( on_stdout = stdout_progress_handler(plan.operation_id, label, on_stdout) if on_stdout is not None: run_options["on_stdout"] = on_stdout + if resource_monitor is not None: + run_options["on_tick"] = resource_monitor.check try: try: result = self.runner.run(plan, **run_options) finally: if after_run is not None: after_run() + if resource_monitor is not None: + resource_monitor.check() if finalize is not None and result.status is OperationStatus.SUCCEEDED: finalize(plan, result) - except BaseException: + except BaseException as exc: # Source is a read-only copy participant. Only the destination # can be uncertain after a failed or interrupted dispatch. for repository, lease in zip(plans, leases): if repository != name: lease.finish(OperationStatus.FAILED, None) + if isinstance(exc, ResourceLimitError): + raise ResourceBudgetExceeded({ + "resource_budget_exceeded": True, "reason": str(exc), + "repository": name, "operation_id": plan.operation_id, + "local_complete": False, "cloud_complete": False, + "replayed": False, "resource_observation": resource_monitor.result(), + }) from None raise if mutating and result.status is not OperationStatus.SUCCEEDED: # These outcomes can leave repository state partially changed. @@ -322,31 +791,146 @@ def _job(self, name): except StopIteration: raise ServiceError("Unknown job") from None - def capture(self, job_name, *, cancel=None): + def capture(self, job_name, *, cancel=None, + quiet_administrative_window_confirmed=False): job = self._job(job_name) + remote_primary = self.repository_roles[job.repository] == "remote-primary" + if remote_primary: + self._operation_environment(job.repository) sources = tuple(source for source in self.sources if source.name in job.sources) + covered = job.coverage_policy is not None + coverage_exclusions = () + sqlite_capture_exclusions = () + coverage_sqlite = {} + coverage_postgresql = {} + validate_coverage = None + if covered: + from .linux_coverage import check_coverage + from .postgresql_coverage import PostgreSQLCoverageContext + if self.policy_bytes is None or self.binding_bytes is None: + raise ServiceError("Covered capture requires exact enrolled policy and binding snapshots") + declaration = self.bindings.coverage_declarations.get(job.name) + if declaration is None: + raise ServiceError("Covered capture declaration is missing") + pg_sources = {source.name: source for source in sources if source.kind == "postgresql"} + pg_context = (PostgreSQLCoverageContext( + pg_sources, self.bindings.database_bindings, self.runner, cancel=cancel, + quiet_administrative_window=quiet_administrative_window_confirmed) + if pg_sources else None) + coverage_arguments = {"state": self.state, "policy_bytes": self.policy_bytes, + "binding_bytes": self.binding_bytes, + "postgresql_context": pg_context} + coverage = check_coverage(declaration, job.coverage_policy, **coverage_arguments) + if not coverage.capture_ready: + raise ServiceError("Covered capture is not ready; inspect and review current coverage") + expected_items = [(source.name, Path(path), source.kind) + for source in sources for path in source.paths] + expected = set(expected_items) + approved_items = [(source_id, path, adapter) + for _, path, source_id, adapter in coverage.approved_roots] + approved = set(approved_items) + if (any(source.kind not in ("files", "sqlite", "postgresql") for source in sources) + or len(expected_items) != len(expected) + or len(approved_items) != len(approved) + or expected != approved): + raise ServiceError("Covered capture sources differ from reviewed root mappings") + pg_by_source = {item["source_id"]: item for item in coverage.approved_postgresql} + if (len(pg_by_source) != len(coverage.approved_postgresql) + or set(pg_by_source) != set(pg_sources)): + raise ServiceError("Covered PostgreSQL sources differ from reviewed logical roots") + for name, reviewed in pg_by_source.items(): + source = pg_sources[name] + if (reviewed["connection"] != source.connection + or reviewed["database"] != source.database + or not reviewed.get("baseline_digest") + or not reviewed.get("data_directory_identity") + or not reviewed.get("recovery_artifacts")): + raise ServiceError("Covered PostgreSQL selector or evidence differs") + coverage_postgresql[name] = dict(reviewed) + physical_roots = {root_id: (path, source_id, adapter) + for root_id, path, source_id, adapter in coverage.approved_roots} + for reviewed in pg_by_source.values(): + for artifact in reviewed["recovery_artifacts"]: + route = physical_roots.get(artifact.get("root_id")) + path = Path(artifact["path"]) + if (route is None or route[2] != "files" + or path != route[0] and route[0] not in path.parents + or not any(item.name == route[1] and item.kind == "files" + and str(route[0]) in item.paths for item in sources)): + raise ServiceError("PostgreSQL recovery artifact is outside reviewed file inputs") + for covered_source in coverage_postgresql.values(): + covered_source["recovery_file_roots"] = [ + {"root_id": root_id, "path": str(path)} + for root_id, (path, _, adapter) in physical_roots.items() + if adapter == "files" and any( + artifact["root_id"] == root_id + for artifact in covered_source["recovery_artifacts"])] + if pg_sources: + approved_pg = coverage.approved_postgresql + baseline = coverage.baseline_digest + candidate = coverage.candidate_digest + + def validate_coverage(): + current = check_coverage(declaration, job.coverage_policy, + **coverage_arguments) + if (not current.capture_ready or current.baseline_digest != baseline + or current.candidate_digest != candidate + or current.approved_postgresql != approved_pg): + raise ServiceError("Reviewed PostgreSQL coverage changed during capture") + return current + coverage_exclusions = coverage.approved_exclusions + sqlite_capture_exclusions = coverage.sqlite_capture_exclusions + sqlite_by_source = {item["source_id"]: item for item in coverage.approved_sqlite} + if (len(sqlite_by_source) != len(coverage.approved_sqlite) + or set(sqlite_by_source) != {source.name for source in sources + if source.kind == "sqlite"}): + raise ServiceError("Covered SQLite sources differ from reviewed root mappings") + for source_name, reviewed in sqlite_by_source.items(): + source = next(item for item in sources if item.name == source_name) + if (source.kind != "sqlite" or source.paths != (reviewed["path"],) + or not reviewed.get("identity") or not reviewed.get("recipe_digest") + or not reviewed.get("baseline_digest")): + raise ServiceError("Covered SQLite sources differ from reviewed root mappings") + coverage_sqlite[source_name] = reviewed + if remote_primary and any(source.kind == "docker" for source in sources): + raise ServiceError("Remote-primary Docker capture has no bounded export support") + monitor = None protected = [self.state] + if remote_primary: + protected.append(self.bindings.staging_dir.resolve()) for binding in self.bindings.repositories.values(): if binding.repository.startswith("/"): protected.append(Path(binding.repository).resolve()) - protected.extend(Path(path).resolve() for path in self.bindings.sensitive_paths) + if not covered: + protected.extend(Path(path).resolve() for path in self.bindings.sensitive_paths) for source in sources: for raw in source.paths: if any(_overlap(Path(raw).resolve(), path) for path in protected): raise ServiceError("Capture source overlaps repository, runtime state, or credentials") - local = self.bindings.repositories[job.repository].repository - if not local.startswith("/"): - raise ServiceError("Local-first jobs require an absolute local repository path") - ancestor = Path(local) - while not ancestor.exists(): - ancestor = ancestor.parent - if shutil.disk_usage(ancestor).free < 64 * 1024 * 1024: - raise ServiceError("Insufficient local space; existing recovery points retained") + if not remote_primary: + local = self.bindings.repositories[job.repository].repository + if not local.startswith("/"): + raise ServiceError("Local-first jobs require an absolute local repository path") + ancestor = Path(local) + while not ancestor.exists(): + ancestor = ancestor.parent + if shutil.disk_usage(ancestor).free < 64 * 1024 * 1024: + raise ServiceError("Insufficient local space; existing recovery points retained") engine = self._engine(job.repository) capture_id = uuid.uuid4().hex receipt = {} + required_replicas = {name: repository_key(self._identity(name)) for name in job.replicas} + artifacts = None + if remote_primary: + artifacts = DurableCaptureArtifacts( + self.bindings.staging_dir, self.state, + repository_key(self._identity(job.repository)), capture_id, job.name) + artifacts.record["required_replicas"] = required_replicas + self.ledger.mark_capture_artifacts(artifacts.record["repository_id"], capture_id) def finalize_capture(plan, result): + if validate_coverage is not None: + validate_coverage() tail = result.stdout_tail if isinstance(tail, bytes): tail = tail.decode("utf-8", errors="replace") @@ -365,26 +949,47 @@ def finalize_capture(plan, result): (repository, snapshot, job, operation, capture_token, required_replicas) VALUES (?, ?, ?, ?, ?, ?)""", (job.repository, snapshot, job.name, plan.operation_id, capture_id, - json.dumps({"schema_version": 1, "bindings": { - name: repository_key(self._identity(name)) for name in job.replicas - }}, sort_keys=True, separators=(",", ":"))), + json.dumps({"schema_version": 1, "bindings": required_replicas}, + sort_keys=True, separators=(",", ":"))), ) receipt["snapshot"] = snapshot + if artifacts is not None: + artifacts.cleanup_success() - with ExitStack() as capture_stack: + with ExitStack() as resource_stack, ExitStack() as capture_stack: + if remote_primary: + monitor = self._resource_monitor(job.repository) + resource_stack.callback(monitor.close) try: paths = capture_stack.enter_context(capture_sources( sources, - self.state, + self.bindings.staging_dir if remote_primary else self.state, timeout=job.timeout_seconds, cancel=cancel, database_bindings=self.bindings.database_bindings, docker_runtime=self.docker_runtime, protected_paths=tuple(protected), operation_id=capture_id, + resource_monitor=monitor, + coverage_sqlite=coverage_sqlite, + coverage_postgresql=coverage_postgresql, + validate_coverage=validate_coverage, + durable_artifacts=artifacts, )) + except ResourceLimitError as exc: + raise ResourceBudgetExceeded({ + "resource_budget_exceeded": True, "reason": str(exc), + "repository": job.repository, "operation_id": capture_id, + "capture_token": capture_id, + "local_complete": False, "cloud_complete": False, + "replayed": False, "resource_observation": monitor.result(), + }) from None except DockerRecoveryError as exc: raise ServiceError(str(exc)) from None + except (CaptureError, OSError) as exc: + if remote_primary: + raise ServiceError(f"Remote capture failed before dispatch; retained capture token {capture_id}") from exc + raise lifecycle = capture_stack.pop_all() def finish_lifecycle(): @@ -395,7 +1000,21 @@ def finish_lifecycle(): try: args = [*engine.base_args(), "backup", "--tag", f"bbackup-job={job.name}", - "--tag", f"bbackup-capture={capture_id}", "--", *paths] + "--tag", f"bbackup-capture={capture_id}"] + if coverage_sqlite: + from .linux_coverage import CoverageError, verify_sqlite_identity + try: + for reviewed in coverage_sqlite.values(): + verify_sqlite_identity(Path(reviewed["path"]), reviewed["identity"]) + except CoverageError: + raise ServiceError("Reviewed SQLite source identity changed before capture dispatch") from None + if validate_coverage is not None: + validate_coverage() + if coverage_exclusions or sqlite_capture_exclusions: + from .linux_coverage import restic_literal_exclusion + for excluded in dict.fromkeys((*coverage_exclusions, *sqlite_capture_exclusions)): + args.extend(("--exclude", restic_literal_exclusion(excluded))) + args.extend(("--", *paths)) plan, _ = self._execute( job.repository, args, @@ -403,17 +1022,330 @@ def finish_lifecycle(): cancel=cancel, operation_id=capture_id, mutating=True, + resource_monitor=monitor, finalize=finalize_capture, after_run=finish_lifecycle, timeout_seconds=job.timeout_seconds, ) + except Exception as exc: + if remote_primary: + if isinstance(exc, ResourceBudgetExceeded): + exc.result["capture_token"] = capture_id + raise + raise ServiceError(f"Remote capture is incomplete; retained capture token {capture_id}") from exc + raise finally: finish_lifecycle() snapshot = receipt["snapshot"] - return {"operation_id": plan.operation_id, "repository": job.repository, "snapshot_id": snapshot, - "local_complete": True, "required_replicas": list(job.replicas), "cloud_complete": not job.replicas} + response = {"operation_id": plan.operation_id, "repository": job.repository, "snapshot_id": snapshot, + "primary_destination": {"repository": job.repository, + "kind": "remote" if remote_primary else "local"}, + "local_complete": not remote_primary, "required_replicas": list(job.replicas), + "cloud_complete": remote_primary, "required_replicas_complete": not job.replicas} + if monitor is not None: + response["resource_observation"] = monitor.result() + return response + + def reconcile_run(self, repository, operation_id, outcome, *, + quiescent_window_confirmed=False, cancel=None): + """Inspect one remote capture under its staging lease before resolving it.""" + + repository_id = repository_key(self._identity(repository)) + ledger_row = self.ledger.inspect(repository_id, operation_id) + if self.repository_roles[repository] != "remote-primary" or ( + ledger_row is not None and ledger_row[0] != "capture"): + self.ledger.reconcile(repository_id, operation_id, outcome) + return {"operation_id": operation_id, "reviewed_outcome": outcome.value, + "replayed": False} + if not re.fullmatch(r"[a-f0-9]{32}", operation_id): + raise ServiceError("Remote capture token is invalid") + try: + artifacts = DurableCaptureArtifacts.load( + self.bindings.staging_dir, self.state, repository_id, operation_id) + except FileNotFoundError: + if (ledger_row is not None and ledger_row[0] == "capture" + and not self.ledger.has_capture_artifact_marker(repository_id, operation_id)): + self.ledger.reconcile(repository_id, operation_id, outcome) + return {"operation_id": operation_id, "reviewed_outcome": outcome.value, + "replayed": False, "legacy_capture": True} + raise ServiceError("New remote capture artifact record is missing") from None + except (OSError, ValueError, CaptureError): + raise ServiceError("Remote capture artifact record is missing or invalid") from None + if quiescent_window_confirmed is not True: + raise ServiceError("Remote capture reconciliation requires a confirmed quiescent repository window") + if ledger_row is None and outcome is ReconciliationOutcome.SUCCEEDED: + raise ServiceError("Capture has no dispatch record; success cannot be reconciled") + with self._resource_monitor(repository, read_only_reconcile=True): + try: + artifacts.recover_empty_preparation() + except (OSError, CaptureError): + raise ServiceError("Remote capture preparation is not an empty owned stage") from None + staging = self.bindings.staging_dir + expected = {".bbackup-staging.lock"} + if artifacts.directory.exists() or (artifacts.record["phase"] not in ( + "cleanup_started", "cleaned") and artifacts.record["stage_inode"] is not None): + expected.add(artifacts.directory.name) + if {path.name for path in staging.iterdir()} != expected: + raise ServiceError("Remote staging contains foreign or missing entries") + try: + artifacts.validate() + except (OSError, KeyError, TypeError, CaptureError): + raise ServiceError("Remote capture artifacts failed validation") from None + engine = self._engine(repository) + tag = f"bbackup-capture={operation_id}" + + def inspect(argv): + plan = OperationPlan(operation_id, repository_id, "inspect", tuple(argv), + env=self._operation_environment(repository), timeout_seconds=3600) + result = self.runner.run(plan, cancel_event=cancel) + if (result.status is not OperationStatus.SUCCEEDED or result.exit_code != 0 + or result.stdout_bytes != len(result.stdout_tail)): + raise ServiceError("Remote capture inspection was incomplete") + return result.stdout_tail + + def require_no_repository_locks(): + # `list locks` does not create a Restic lock. A clear listing is + # only an observation; the caller must keep writers quiescent. + if inspect([*engine.base_args(), "--no-lock", "list", "locks"]): + raise ServiceError("Remote repository has an active Restic lock") + + def inspect_export(snapshot_id, item, *, return_bytes=False): + expected_size = item["size"] + expected_hash = item["sha256"] + observed_size = 0 + digest = hashlib.sha256() + captured = bytearray() if return_bytes else None + + def receive(chunk): + nonlocal observed_size + observed_size += len(chunk) + if observed_size > expected_size: + raise ServiceError("Remote capture export exceeds the recorded size") + digest.update(chunk) + if captured is not None: + captured.extend(chunk) + + plan = OperationPlan( + operation_id, repository_id, "inspect", + tuple([*engine.base_args(), "--no-lock", "dump", snapshot_id, + str(artifacts.directory / item["name"])]), + env=self._operation_environment(repository), timeout_seconds=3600, + ) + result = self.runner.run(plan, cancel_event=cancel, + on_stdout_chunk=receive) + if (result.status is not OperationStatus.SUCCEEDED or result.exit_code != 0 + or result.stdout_bytes != observed_size or observed_size != expected_size + or digest.hexdigest() != expected_hash or result.stdout_tail): + raise ServiceError("Remote capture export differs from the retained record") + return bytes(captured) if captured is not None else None + + def inspect_covered_material(snapshot_id, manifest_bytes): + try: + manifest_data = strict_json(manifest_bytes) + if (manifest_data["schema_version"] != 2 + or not isinstance(manifest_data["exports"], list)): + raise ValueError() + covered = [entry for entry in manifest_data["exports"] + if isinstance(entry, dict) and entry.get("coverage") is not None] + if not covered: + return + snapshot_data = strict_json(inspect( + [*engine.base_args(), "--no-lock", "cat", "snapshot", snapshot_id])) + paths = snapshot_data["paths"] + if not isinstance(paths, list): + raise ValueError() + for entry in covered: + if entry.get("kind") != "postgresql": + raise ValueError() + refs = _covered_postgresql_refs( + entry, source=entry["source"], snapshot_paths=paths) + for ref in refs: + observed_size = 0 + digest = hashlib.sha256() + + def receive(chunk): + nonlocal observed_size + observed_size += len(chunk) + if observed_size > 64 * 1024 * 1024: + raise ServiceError("Covered PostgreSQL recovery artifact exceeds bound") + digest.update(chunk) + + plan = OperationPlan( + operation_id, repository_id, "inspect", + tuple([*engine.base_args(), "--no-lock", "dump", snapshot_id, + ref["path"]]), + env=self._operation_environment(repository), timeout_seconds=3600, + ) + result = self.runner.run(plan, cancel_event=cancel, + on_stdout_chunk=receive) + if (result.status is not OperationStatus.SUCCEEDED or result.exit_code != 0 + or result.stdout_bytes != observed_size or result.stdout_tail + or digest.hexdigest() != ref["sha256"]): + raise ServiceError("Covered PostgreSQL recovery artifact differs from reviewed manifest") + except (ValueError, KeyError, TypeError, UnicodeDecodeError): + raise ServiceError("Covered PostgreSQL recovery manifest is invalid") from None + + try: + require_no_repository_locks() + inventory = strict_json(inspect([*engine.base_args(), "--no-lock", "snapshots", "--tag", tag])) + except (ValueError, UnicodeDecodeError, TypeError): + raise ServiceError("Remote capture inventory is invalid") from None + if not isinstance(inventory, list) or len(inventory) > 1: + raise ServiceError("Remote capture tag is ambiguous") + snapshot = None + if inventory: + item = inventory[0] + if (not isinstance(item, dict) or not isinstance(item.get("tags"), list) + or tag not in item["tags"] + or f"bbackup-job={artifacts.record['job']}" not in item["tags"]): + raise ServiceError("Remote capture tag does not match the record") + snapshot = _snapshot_id(item.get("id")) + with self._receipts() as db: + recorded = db.execute( + "SELECT snapshot FROM snapshots WHERE repository=? AND capture_token=?", + (repository, operation_id)).fetchall() + if len(recorded) > 1 or (recorded and (snapshot is None or recorded[0][0] != snapshot)): + raise ServiceError("Remote capture receipt conflicts with repository inspection") + if outcome is ReconciliationOutcome.SUCCEEDED: + if snapshot is None or artifacts.record["phase"] not in ( + "ready", "cleanup_started", "cleaned"): + raise ServiceError("Remote capture success is not proven") + manifest = artifacts.record["manifest"] + for export in artifacts.record["exports"]: + inspect_export(snapshot, export) + if manifest is not None: + inspect_covered_material(snapshot, inspect_export(snapshot, manifest, return_bytes=True)) + require_no_repository_locks() + replicas = artifacts.record["required_replicas"] + if not isinstance(replicas, dict): + raise ServiceError("Remote capture replica record is invalid") + required = json.dumps({"schema_version": 1, "bindings": replicas}, + sort_keys=True, separators=(",", ":")) + with self._receipts() as db: + existing = db.execute( + "SELECT job,operation,capture_token,required_replicas FROM snapshots " + "WHERE repository=? AND snapshot=?", (repository, snapshot)).fetchone() + if existing is None: + db.execute("INSERT INTO snapshots " + "(repository,snapshot,job,operation,capture_token,required_replicas) " + "VALUES (?,?,?,?,?,?)", + (repository, snapshot, artifacts.record["job"], operation_id, + operation_id, required)) + elif tuple(existing) != (artifacts.record["job"], operation_id, + operation_id, required): + raise ServiceError("Remote capture receipt conflicts with the snapshot") + try: + artifacts.cleanup_success() + except (OSError, KeyError, TypeError, CaptureError): + raise ServiceError("Remote capture cleanup is incomplete; retry reconciliation") from None + elif snapshot is not None: + raise ServiceError("Remote capture snapshot exists; failure is not proven") + else: + require_no_repository_locks() + if artifacts.record["phase"] in ("cleanup_started", "cleaned"): + raise ServiceError("Remote capture cleanup already began after success") + artifacts.reviewed_failure(outcome.value) + if ledger_row is not None: + self.ledger.reconcile(repository_id, operation_id, outcome) + return {"operation_id": operation_id, "reviewed_outcome": outcome.value, + "snapshot_id": snapshot, "retained_exports": outcome is not ReconciliationOutcome.SUCCEEDED, + "artifact_record_sha256": artifacts.record_sha256(), + "replayed": False} + + def retained_exports(self, repository, *, limit=100, offset=0): + """List bounded, declared private artifact summaries without provider access.""" + + if type(limit) is not int or not 1 <= limit <= 1000 or type(offset) is not int or offset < 0: + raise ServiceError("Invalid pagination") + if self.repository_roles.get(repository) != "remote-primary": + raise ServiceError("Retained exports require a remote-primary repository") + repository_id = repository_key(self._identity(repository)) + directory = self.state / "capture-artifacts" + if not directory.exists(): + return [] + info = directory.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise ServiceError("Private capture record directory changed") + summaries = [] + for path in sorted(directory.iterdir()): + match = re.fullmatch(r"([a-f0-9]{32})\.json", path.name) + if match is None: + continue + token = match.group(1) + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(descriptor) + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or info.st_mode & 0o077 or info.st_size > 1024 * 1024): + raise ServiceError("Private capture record changed") + with os.fdopen(descriptor, "r", encoding="utf-8", closefd=False) as stream: + record = json.load(stream) + finally: + os.close(descriptor) + if not isinstance(record, dict): + raise ServiceError("Private capture record is invalid") + if record.get("repository_id") != repository_id: + continue + artifact = DurableCaptureArtifacts.load( + self.bindings.staging_dir, self.state, repository_id, token) + if artifact.record["phase"] in ("cleaned", "discarded"): + continue + exports = artifact.record["exports"] + if (not isinstance(exports, list) or any( + not isinstance(item, dict) or type(item.get("size")) is not int + or item["size"] < 0 for item in exports)): + raise ServiceError("Private capture record is invalid") + summaries.append({ + "repository": repository, "operation_id": token, + "phase": artifact.record["phase"], + "reviewed_outcome": artifact.record.get("reviewed_outcome"), + "completed_export_count": len(exports), + "completed_export_bytes": sum(item["size"] for item in exports), + "artifact_record_sha256": artifact.record_sha256(), + "file_hashes_verified": False, + }) + except (OSError, ValueError, KeyError, TypeError, CaptureError): + raise ServiceError("Private capture record is invalid") from None + return summaries[offset:offset + limit] + + def discard_exports(self, repository, operation_id, artifact_record_sha256): + """Explicitly release a reviewed failed capture's exact private stage.""" + + if self.repository_roles.get(repository) != "remote-primary": + raise ServiceError("Retained exports require a remote-primary repository") + repository_id = repository_key(self._identity(repository)) + try: + artifact = DurableCaptureArtifacts.load( + self.bindings.staging_dir, self.state, repository_id, operation_id) + except (OSError, ValueError, CaptureError): + raise ServiceError("Private capture record is missing or invalid") from None + row = self.ledger.inspect(repository_id, operation_id) + reviewed = artifact.record.get("reviewed_outcome") + if (reviewed not in ("failed", "abandoned") + or row is not None and row != ( + "capture", OperationStatus.RECONCILED_FAILED if reviewed == "failed" + else OperationStatus.RECONCILED_ABANDONED)): + raise ServiceError("Capture failure has not been explicitly reconciled") + with self._resource_monitor(repository, read_only_reconcile=True): + staging = self.bindings.staging_dir + expected = {".bbackup-staging.lock"} + if artifact.directory.exists() or artifact.record["phase"] not in ( + "discard_started", "discarded"): + expected.add(artifact.directory.name) + if {path.name for path in staging.iterdir()} != expected: + raise ServiceError("Remote staging contains foreign or missing entries") + try: + artifact.discard(artifact_record_sha256) + except (OSError, ValueError, KeyError, TypeError, CaptureError): + raise ServiceError("Capture exports changed; explicit disposal was refused") from None + return {"repository": repository, "operation_id": operation_id, + "discarded_export_count": len(artifact.record["exports"]), + "artifact_record_sha256": artifact.record_sha256(), "replayed": False} def copy(self, source, destination, snapshot, *, cancel=None, timeout_seconds=3600): + if any(self.repository_roles.get(name) == "remote-primary" for name in (source, destination)): + raise ServiceError("Remote-primary copies do not yet have bounded resource support") snapshot = _snapshot_id(snapshot) if source == destination or self._identity(source) == self._identity(destination): raise ServiceError("Copy requires independent repositories") @@ -1111,6 +2043,9 @@ def restore_database(self, repository, snapshot, source, connection, target_data raise ValueError() except (ValueError, TypeError, KeyError): raise ServiceError("Snapshot does not contain a valid selected native database export") from None + recovery_artifacts = (_covered_postgresql_refs( + entry, source=source, snapshot_paths=metadata["paths"]) + if binding.kind == "postgresql" else []) # The bounded runner streams the archive directly to private temporary # storage. No original-host source configuration or ledger is required. with tempfile.TemporaryDirectory(prefix="database-restore-", dir=self.state) as staging: @@ -1119,6 +2054,30 @@ def restore_database(self, repository, snapshot, source, connection, target_data archive.chmod(0o600) self._execute(repository, [*engine.base_args(), "dump", snapshot, str(archive_path)], label="database-export", cancel=cancel, on_stdout=output.write) + if recovery_artifacts: + archive_hash = hashlib.sha256() + with archive.open("rb") as exported: + for chunk in iter(lambda: exported.read(1024 * 1024), b""): + archive_hash.update(chunk) + if archive_hash.hexdigest() != entry["export_sha256"]: + raise ServiceError("Covered PostgreSQL export digest differs from reviewed manifest") + for artifact in recovery_artifacts: + digest = hashlib.sha256() + size = 0 + + def hash_artifact(chunk): + nonlocal size + size += len(chunk) + if size > 64 * 1024 * 1024: + raise ServiceError("Covered PostgreSQL recovery artifact exceeds bound") + digest.update(chunk) + + _, recovered = self._execute(repository, + [*engine.base_args(), "dump", snapshot, artifact["path"]], + label="database-material", cancel=cancel, on_stdout=hash_artifact) + if (recovered.stdout_bytes != size + or digest.hexdigest() != artifact["sha256"]): + raise ServiceError("Covered PostgreSQL recovery artifact digest differs") client = "pg_restore" if binding.kind == "postgresql" else binding.kind plan = OperationPlan.create(identity, "database-restore", [client]) with self.ledger.begin(plan) as lease: @@ -1144,8 +2103,10 @@ def restore_database(self, repository, snapshot, source, connection, target_data return {"operation_id": plan.operation_id, "snapshot_id": snapshot, "database_validated": True, "application_validated": False, "target_database": result} - def run_job(self, job_name, *, cancel=None): - result = self.capture(job_name, cancel=cancel) + def run_job(self, job_name, *, cancel=None, + quiet_administrative_window_confirmed=False): + result = self.capture(job_name, cancel=cancel, + quiet_administrative_window_confirmed=quiet_administrative_window_confirmed) result["replicas"] = [] result["replica_failures"] = [] for destination in self._job(job_name).replicas: @@ -1158,12 +2119,17 @@ def run_job(self, job_name, *, cancel=None): result["repository"], destination, result["snapshot_id"], cancel=cancel, timeout_seconds=self._job(job_name).timeout_seconds, )) + if self.bindings.repositories.get(destination) is not None: + if self.bindings.repositories[destination].repository.startswith("/"): + result["local_complete"] = True + else: + result["cloud_complete"] = True except (ServiceError, RuntimeError, OSError, sqlite3.Error, ValueError): result["replica_failures"].append({"destination_repository": destination, "reason": "copy_failed_or_uncertain"}) if cancel is not None and cancel.is_set(): break - result["cloud_complete"] = not result["replica_failures"] + result["required_replicas_complete"] = not result["replica_failures"] if result["replica_failures"]: raise ReplicaJobError(result) return result diff --git a/bbackup/service_status.py b/bbackup/service_status.py index 1e065b3..fd0b8c5 100644 --- a/bbackup/service_status.py +++ b/bbackup/service_status.py @@ -46,6 +46,7 @@ class CheckpointStatus(TypedDict): class OperationalStatus(TypedDict): last_local_capture: str | None + last_remote_primary_capture: str | None last_independent_copy: str | None last_cloud_copy: str | None last_repository_check: str | None @@ -239,7 +240,8 @@ def latest(label: str, *, allowed: set[str] | None = None) -> str | None: now = datetime.now(timezone.utc) return { - "last_local_capture": latest("capture", allowed=identities), + "last_local_capture": latest("capture", allowed=identities - cloud_identities), + "last_remote_primary_capture": latest("capture", allowed=cloud_identities), "last_independent_copy": latest("copy", allowed=identities), "last_cloud_copy": latest("copy", allowed=cloud_identities), "last_repository_check": latest("check", allowed=identities), diff --git a/docs/AGENT_INTEGRATION.md b/docs/AGENT_INTEGRATION.md index f38d971..cf0038c 100644 --- a/docs/AGENT_INTEGRATION.md +++ b/docs/AGENT_INTEGRATION.md @@ -1,6 +1,6 @@ # Agent integration -The version 2 command line is designed for agents and other noninteractive clients. It accepts strict JSON input, publishes a schema for each operation, and emits sanitized versioned results. Stable v2.0.2 qualification is limited to the tested local/file, selected live database, headless dashboard, and AMD64 Ubuntu paths listed below. B2/AWS and native Docker are not production-qualified; follow the same read-before-write and no-replay rules you would use for an operator. +The version 2 command line is designed for agents and other noninteractive clients. It accepts strict JSON input, publishes a schema for each operation, and emits sanitized versioned results. Stable v2.0.4 qualification retains the tested local/file, selected live database, headless dashboard, and AMD64 Ubuntu scope listed below. New cloud and Linux workflows ship as experimental and remain unqualified; B2/AWS and native Docker are not production-qualified. Follow the same read-before-write and no-replay rules you would use for an operator. ## Discover commands and schemas @@ -48,6 +48,18 @@ bbackup production \ Restore requires a new target directory and verifies file data. It does not validate application startup or database behavior. Database restores accept trusted PostgreSQL, MySQL, and MariaDB archives with database-specific destination safeguards: PostgreSQL creates a new database using snapshot-time locale metadata; MySQL/MariaDB require the original database name on a separate empty recovery server with only built-in schemas. Set the explicit `trusted_archive` acknowledgement. Never restore untrusted archives: database restores execute archived SQL. See [the v2 database restore limits](development/version-2.md#native-database-capture-and-restore). +Reviewed Linux coverage also has an experimental manual PostgreSQL path in +v2.0.4; it does not change ordinary PostgreSQL adapters or stable +qualification. `jobs run`, `coverage discover`, `coverage review`, +`coverage check`, and `coverage recovery-evidence` accept the boolean +`--quiet-administrative-window-confirmed`, which defaults to false. Set it only +while an externally coordinated window prevents administrator, schema, role, +database, and configuration changes; it does not stop those changes itself. +Ordinary row writes may continue. Scheduled covered PostgreSQL jobs remain +blocked because timer dispatch cannot confirm the window. See the [manual +coverage procedure](development/version-2.md#experimental-postgresql-covered-capture) +for declaration, capture, and restore limits. + ## Read bounded operation records Use pagination for operation events and snapshot receipts: @@ -66,7 +78,21 @@ bbackup production --config "$HOME/.config/bbackup/config.json" \ ## Reconcile uncertain mutations -If a mutating command returns `requires_reconciliation` or the connection is lost after dispatch, do not repeat the command. Read the repository or destination state, inspect the operation ID, and decide what actually completed. Reconciliation records an administrator-reviewed outcome; it does not rerun the command or repair missing receipts. +If a mutating command returns `requires_reconciliation` or the connection is lost after dispatch, do not repeat the command. Read the repository or destination state, inspect the operation ID, and decide what actually completed. Reconciliation records an administrator-reviewed outcome without rerunning the command. + +For retained remote database exports, successful reconciliation verifies the +snapshot and each export's actual bytes before repairing the capture receipt +and releasing its owned staging files. A failed outcome retains the exports. +Remote capture reconciliation also requires `quiescent_window_confirmed: true` +after stopping repository writers for the entire inspection. It uses Restic's +`--no-lock` for reads and refuses observed repository locks; those observations +cannot prevent a new writer from starting elsewhere. +`runs retained-exports` discovers bounded capture IDs, declared counts and record +digests, including captures stopped before dispatch with no ledger row. +`runs discard-exports` explicitly discards reviewed failed exports using the exact +current record digest. See the [remote-primary recovery workflow](cloud-storage.md#remote-primary-with-bounded-staging-experimental) +for persistent-staging requirements and disposal steps. `runs list` remains a +ledger event stream. ```bash bbackup production \ diff --git a/docs/PUBLISHING_CHECKLIST.md b/docs/PUBLISHING_CHECKLIST.md index eb29fff..ce6dc10 100644 --- a/docs/PUBLISHING_CHECKLIST.md +++ b/docs/PUBLISHING_CHECKLIST.md @@ -24,7 +24,8 @@ Use this checklist before changing public project metadata or publishing a relea ## Version 2 tested-scope claims -- [ ] Describe v2.0.3 as qualified only for local file capture/restore, live PostgreSQL 17.11 / MySQL 8.4.12 / MariaDB 11.8.9 capture/restore, headless dashboard happy paths, and AMD64 Ubuntu 24.04/26.04 packages. +- [ ] Describe v2.0.4 as qualified only for local file capture/restore, live PostgreSQL 17.11 / MySQL 8.4.12 / MariaDB 11.8.9 capture/restore, headless dashboard happy paths, and AMD64 Ubuntu 24.04/26.04 packages; this is the v2.0.3 tested core scope. +- [ ] Describe new v2.0.4 cloud and Linux workflows as experimental and unqualified; keep requests #23–#26 open and do not claim a guaranteed thirty-day window, off-host custody, final remote-server qualification, or complete-machine recovery. - [ ] Keep B2/AWS, native Docker, ARM64 backup/restore, cross-application-version upgrades/rollback, root-owned systemd package templates, and cloud/off-host protection explicitly unqualified; describe the separate user-timer guest drills with their exact limits. - [ ] State that ARM64 packages may be built by the release matrix but are not functionally qualified by package-install smoke alone. - [ ] Keep native B2 capability inspection distinct from backup transfer; inspection is read-only and always reports `protection_qualified: false`. diff --git a/docs/README.md b/docs/README.md index 8373481..a69d0f5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,7 @@ bbackup is for Linux administrators, homelab and self-hosted operators, and small infrastructure teams who need encrypted local backups, verified file restores, and visible scheduled jobs. Choose a guide by the work you are doing. -These guides describe v2.0.3. The [qualification record](development/version-2.md) is the authority for the exact qualified scope, version-specific evidence, and remaining limits; an implemented path alone does not establish support. Operators still using v2.0.2 can follow its [versioned documentation](https://github.com/CruxExperts/BestBackup/tree/v2.0.2/docs) until they migrate. +These guides describe v2.0.4. The [qualification record](development/version-2.md) is the authority for the exact qualified scope, version-specific evidence, and remaining limits; an implemented path alone does not establish support. The v2.0.4 release retains the v2.0.3 tested core scope, while new cloud and Linux workflows are experimental and unqualified. Operators still using v2.0.2 can follow its [versioned documentation](https://github.com/CruxExperts/BestBackup/tree/v2.0.2/docs) until they migrate. The [release dependency evidence](development/release-dependencies.json) records the reviewed versions, source timestamps, lock hash, and audit limits. @@ -14,7 +14,8 @@ The [release dependency evidence](development/release-dependencies.json) records | Migrate an existing schema-2 setup | [Quick start: migrate an existing configuration](../QUICKSTART.md#migrate-an-existing-schema-2-configuration) | | Maintain an older Docker/YAML workflow | [Legacy management](management.md) and [encryption compatibility](encryption.md) | | Schedule jobs or integrate with scripts and agents | [Quick start: apply a schedule](../QUICKSTART.md#apply-a-schedule-deliberately), [agent integration](AGENT_INTEGRATION.md), and the [production command reference](production-cli.md) | -| Check what v2.0.3 qualifies | [Version 2 qualification record](development/version-2.md) | +| Check what v2.0.4 qualifies | [Version 2 qualification record](development/version-2.md) | +| Evaluate experimental cloud recovery, retention or Linux coverage | [Experimental program and limits](development/version-2.md#experimental-cloud-and-machine-recovery-features) and [cloud storage](cloud-storage.md) | ## Operator references diff --git a/docs/VERSIONING.md b/docs/VERSIONING.md index b62dd33..d2db1aa 100644 --- a/docs/VERSIONING.md +++ b/docs/VERSIONING.md @@ -1,6 +1,6 @@ # Versioning -`VERSION` is the canonical release version. The current version is `2.0.3`. +`VERSION` is the canonical release version. The current version is `2.0.4`. The earlier `v2.0.0` and `v2.0.1` tags did not produce release artifacts; the retained legacy line is `1.8.6`. Stable support is limited to the tested scope in [`docs/development/version-2.md`](development/version-2.md). @@ -17,7 +17,7 @@ Release-facing references must match it before publishing: The release workflow accepts alpha, beta, and release-candidate suffixes. A prerelease tag such as `v2.0.0-alpha.1` is marked as a GitHub prerelease; a -stable tag such as `v2.0.2` is not. Every tag must match `VERSION`, its dated +stable tag such as `v2.0.4` is not. Every tag must match `VERSION`, its dated changelog section, and generated schemas before release validation. Normal release batches use a patch bump by default, including commits with diff --git a/docs/architecture.md b/docs/architecture.md index 86e87e0..e5aa991 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -This page describes the selected stable v2.0.2 tested scope and the retained 1.x implementation. The v2 command is `bbackup production`; legacy 1.x Docker/YAML commands remain available. +This page describes the selected stable v2.0.3 tested scope and the retained 1.x implementation. The v2 command is `bbackup production`; legacy 1.x Docker/YAML commands remain available. ## Version 2 architecture @@ -22,7 +22,16 @@ Cloud recovery kits inventory exact object versions, bind the provider endpoint ### Interfaces and scope -The `bbackup production` CLI is strict and schema-discoverable; the Textual dashboard provides mouse and keyboard job operation. Headless dashboard happy paths are tested. The stable v2.0.2 release scope is local/file capture and restore, the listed live database drills, dashboard happy paths, and AMD64 Ubuntu 24.04/26.04 packages. B2/AWS, native Docker, ARM64 package outputs, cross-application upgrade/rollback, and systemd boot/reboot remain unqualified or deferred. See [version 2 development](development/version-2.md) for release boundaries, [agent integration](AGENT_INTEGRATION.md) for the JSON contract, and [cloud storage](cloud-storage.md) for B2/S3 limits. +The `bbackup production` CLI is strict and schema-discoverable; the Textual dashboard provides mouse and keyboard job operation. Headless dashboard happy paths are tested. The stable v2.0.3 release scope is local/file capture and restore, the listed live database drills, dashboard happy paths, and AMD64 Ubuntu 24.04/26.04 packages. User-timer reboot evidence is limited to the recorded guest drills. B2/AWS, native Docker, ARM64 backup/restore, root system units and off-host recovery remain unqualified or deferred. See [version 2 development](development/version-2.md) for release boundaries, [agent integration](AGENT_INTEGRATION.md) for the JSON contract, and [cloud storage](cloud-storage.md) for B2/S3 limits. + +Unreleased work adds restricted B2 source-key assessment, independent central +acceptance of signed historical checkpoints, centrally authorized noncurrent +version reclamation, remote-primary capture with bounded staging, and reviewed +Linux file and transactional SQLite coverage. These paths use the same operation boundary and existing +repository ledger; coverage review owns a separate exclusive policy lease. +Management credentials remain an explicit option, separate from the default +restricted source role. Their implementation and workstation fixture evidence +do not expand the stable support boundary. The remaining module and configuration sections below describe the retained 1.x Docker/filesystem workflow. Their YAML settings, archive model, and AES/RSA file encryption do not configure the v2 implementation. diff --git a/docs/assets/README.md b/docs/assets/README.md index 2419e1b..c4236a9 100644 --- a/docs/assets/README.md +++ b/docs/assets/README.md @@ -5,8 +5,7 @@ Active README artwork follows the bbackup forest-green identity and keeps all pr ## Active assets - `bbackup-banner-v2.png`: wide README banner, generated with OpenAI's built-in image generation tool on 2026-10-17. -- `bbackup-social-v2.png`: matching 2:1 master image, generated from the banner reference. -- `bbackup-social-v2.jpg`: 1774×887 delivery encoding for GitHub social upload, under 1 MB. EXIF and XMP retain ownership and usage terms. +- `bbackup-social-v3.jpg`: current GitHub social upload, generated with OpenAI's built-in image generation tool on 2026-10-19 using the README banner as a brand reference. Center-cropped and resized to GitHub's recommended 1280×640 dimensions; 154,877 bytes, under 1 MB. EXIF and XMP retain ownership and usage terms. - `bbackup-dashboard-preview.svg`: Textual headless dashboard capture, using fixture data only and illustrative capacity. Regenerate with `uv run python scripts/generate_dashboard_preview.py` (requires restic). @@ -18,11 +17,27 @@ Validate active images with: uv run python scripts/check_assets.py ``` -GitHub requires a PNG, JPG, or GIF under 1 MB; the JPEG is the prepared upload asset. +## Set the GitHub social preview + +GitHub does not automatically use a README hero or an image committed to the repository. +Without a separately uploaded social preview, shared repository links use GitHub's +generated card with the owner's avatar. + +1. Open the repository's **Settings → General → Social preview**. +2. Choose **Edit → Upload an image…** and select [`bbackup-social-v3.jpg`](bbackup-social-v3.jpg). +3. Confirm the preview shows the bbackup wordmark and both headline lines without clipping. + +Uploading is a separate repository-settings change; committing this file does not +perform it. Existing shared links may retain a platform's cached preview after upload. +The README continues to use its wider banner. + +GitHub accepts PNG, JPG, or GIF under 1 MB and recommends 1280×640 for best display. See [GitHub’s social preview guide](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/customizing-your-repositorys-social-media-preview). ## Historical assets +`bbackup-social-v2.png` and `bbackup-social-v2.jpg` are the previous social master and upload encoding, retained with their provenance. Use `bbackup-social-v3.jpg` for new uploads. + `bbackup-banner.svg` is the original editable vector banner. It is retained for provenance and is no longer the README hero. `bbackup-hero.png` is an older unused illustration whose source prompt and generation record are unavailable; do not reuse it as a qualified product diagram or infer a license from the code license. ## Artwork terms diff --git a/docs/assets/bbackup-social-v3.jpg b/docs/assets/bbackup-social-v3.jpg new file mode 100644 index 0000000..a966e00 Binary files /dev/null and b/docs/assets/bbackup-social-v3.jpg differ diff --git a/docs/assets/provenance.json b/docs/assets/provenance.json index 8a2bb5a..55763f9 100644 --- a/docs/assets/provenance.json +++ b/docs/assets/provenance.json @@ -52,6 +52,21 @@ "website": "https://www.cruxexperts.com/", "rights": "Private asset. Property of Crux Experts LLC. Use, reproduction, modification, distribution, or publication without prior written permission is prohibited.", "encoding": "JPEG quality 2 via FFmpeg; composition and dimensions unchanged; XMP and EXIF rights metadata embedded with ExifTool" + }, + { + "path": "docs/assets/bbackup-social-v3.jpg", + "purpose": "GitHub social upload", + "generator": "OpenAI gpt-image-2-codex via built-in image generation", + "created": "2026-10-19", + "prompt": "Generate a premium GitHub social preview for bbackup, using bbackup-banner-v2.png as a style and brand reference. Deep forest-green backdrop, luminous jade glass, brushed silver metal, bone-white bold sans-serif lettering, realistic studio-lit physical snapshot-storage sculpture. Large lowercase bbackup wordmark on the left with exact two-line headline 'Back up with confidence.' / 'Restore with clarity.' Right: compact metal enclosure holding layered emerald snapshot cards, secondary copy slab and luminous restored document, subtle green tracks. Rich tactile materials, controlled reflections, crisp edges, restrained cinematic lighting, editorial simplicity. Keep lettering and important objects in a central safe area for a centered 2:1 crop; expendable dark green background above and below. Sharp correctly spelled text only; no supporting small text, people, avatars, GitHub mascot, cloud logos, locks, fake UI, extra logos, borders, watermarks, badges or feature claims.", + "reference": "bbackup-banner-v2.png", + "dimensions": [1280, 640], + "original_sha256": "f4bb44b15ff8786d6c6a9af9cd7aaa60e864bf04be756e69eac66eb83e42be09", + "sha256": "967a73f3302e4c17e5809e5efff2d21843f936a3cd8f44c5dcc790e18944cce3", + "copyright": "Copyright © 2026 Crux Experts LLC", + "website": "https://www.cruxexperts.com/", + "rights": "Private asset. Property of Crux Experts LLC. Use, reproduction, modification, distribution, or publication without prior written permission is prohibited.", + "encoding": "Generated 1536×1024 WebP; FFmpeg centered 1536×768 crop, Lanczos resize to 1280×640, JPEG quality 2; EXIF and XMP rights metadata embedded with ExifTool." } ] } diff --git a/docs/cli-skills.md b/docs/cli-skills.md index af91b0c..04b74f6 100644 --- a/docs/cli-skills.md +++ b/docs/cli-skills.md @@ -1,6 +1,6 @@ # CLI skills catalog -> Generated from the bbackup/bbman CLI metadata. Version: 2.0.3. This catalog is authoritative for this version. +> Generated from the bbackup/bbman CLI metadata. Version: 2.0.4. This catalog is authoritative for this version. > Source: `bbackup/cli_metadata.py` rendered by `scripts/generate_cli_skills.py`; run the generator's `--check` mode before publishing. ## bbackup diff --git a/docs/cloud-storage.md b/docs/cloud-storage.md index 1f1c1e4..d7aed2b 100644 --- a/docs/cloud-storage.md +++ b/docs/cloud-storage.md @@ -1,15 +1,19 @@ # Cloud storage: Backblaze B2 and Amazon S3 -Keep your first recovery point on local storage. The v2.0.2 tested stable-release -scope includes local/file capture and restore, selected live database drills, +Keep your first recovery point on local storage. The v2.0.4 tested stable-release +scope retains local/file capture and restore, selected live database drills, headless dashboard happy paths, and AMD64 Ubuntu packages. -**Backblaze B2 and Amazon S3 are implemented but not production-qualified.** -No live B2 or AWS capture/check/restore result is recorded. The local Garage S3 -test passed capture, copy, full-data checking, and restore, but does not qualify -B2 or Amazon S3. -Treat the provider setup below as experimental; do not use B2/AWS as a -production recovery path until live provider capture/check/restore drills pass. +**Backblaze B2 and Amazon S3 are implemented but not production-qualified.** A +small live B2 source-path drill passed on 2026-10-20; it does not qualify B2 +protection or production recovery. No live AWS capture/check/restore result is +recorded. The local Garage S3 test also passed capture, copy, full-data checking, +and restore, but does not qualify B2 or Amazon S3. The provider workflows ship +as experimental in v2.0.4; keep B2/AWS outside production recovery until the +remaining provider and independent-recovery evidence is complete. A separate live B2 +lifecycle-overlap guard confirmed only the configuration check and central +retention refusal; it did not test elapsed expiry or physical deletion. See the +[guard evidence](#b2-lifecycle-overlap-guard-experimental). ## Choose a connection @@ -61,6 +65,407 @@ the standard AWS credential variable names in the private process environment. For the native restic route, use the native variable names in the table above. The native SDK check does not change bucket policies or persist authorization. +For central lifecycle inspection, Backblaze documents the separate +`readBucketLifecycleRules` capability for `Get Lifecycle Configuration`. Add +that read capability only to the centrally controlled verifier/retention +identity; the source key does not need it. Do not add +`writeBucketLifecycleRules` for a read-only inspection. See +[Backblaze's S3-compatible app-key capabilities](https://www.backblaze.com/docs/cloud-storage-s3-compatible-app-keys). + +### Restricted source profile (experimental) + +The `restricted_source_profile` result is a versioned assessment of the SDK's +authorization metadata. It requires one exact bucket and a nonempty, +directory-scoped machine prefix, repository read/list/write capabilities, and +no permanent-version deletion, administration, or unknown capabilities. A broad +key can still be inspected, but does not satisfy this profile. An eligible +metadata result alone does not prove that a forbidden operation is denied. The +small live trial below passed seven denial checks, but full recovery +qualification remains required by +[issue #23](https://github.com/CruxExperts/BestBackup/issues/23). + +Use a distinct prefix per machine, such as `machines/MACHINE_ID/`. Keep the +provisioner, central verifier, and retention executor identities off source +machines. Backblaze distinguishes deleting by name using `writeFiles` from +permanently deleting a specific version using `deleteFiles`; a source writer may +hide names or upload replacement versions. This profile does not prevent those +attacks. Recovery depends on exact historical version identities, adequate +noncurrent retention, independent escrow, and successful restoration. See +[Backblaze's S3 app-key capabilities](https://www.backblaze.com/docs/cloud-storage-s3-compatible-app-keys) +and [Object Lock behavior](https://www.backblaze.com/docs/cloud-storage-object-lock). + +A small live B2 source-path trial passed on 2026-10-20 using an exact-bucket, +machine-prefix source key. The trial completed two captures, a repository check, +and restores of the older and newer snapshots; the incremental file was absent +from the older restore and present in the newer one, and both restores preserved +the hidden fixture file and symlink. It also checked the source key's four +reported capabilities and seven denied operations. The detailed evidence and +limits are recorded in the +[version 2 development qualification record](development/version-2.md#live-backblaze-b2-source-trial). +This narrow result does not establish the thirty-day historical grace period, +provider lock/hold behavior, off-host escrow, or production recovery. + +### Central recovery acceptance (experimental) + +On an independent verifier, `recovery accept` opens a source-signed encrypted +recovery kit, reconstructs the exact historical repository versions, and runs a +full Restic data check and exact repository/snapshot identity verification. Only +then does it write a signed central acceptance record into a new private +directory. The verifier fingerprint must differ from the source signer. Trust +that fingerprint through separately controlled configuration and keep its +private signing key and the escrow decryption authority off source hosts. Supply +complete primary-key fingerprints; signing-subkey fingerprints are rejected. + +```bash +bbackup production recovery accept \ + --kit /secure/escrow/recovery-kit.gpg --target /secure/acceptances/NEW_RECORD \ + --source-signer SOURCE_SIGNER_FINGERPRINT --verifier CENTRAL_VERIFIER_FINGERPRINT \ + --repository cloud --repository-id FULL_REPOSITORY_ID --snapshot FULL_SNAPSHOT_ID \ + --bucket YOUR_BUCKET --endpoint https://s3.REGION.backblazeb2.com \ + --prefix machines/MACHINE_ID/ +``` + +Run this with read-only historical-version access and enough private disk for +the reconstructed repository. Use the independently retrieved kit and keys; +the command does not establish their off-host custody. Central verification time +comes from the verifier, not the source snapshot timestamp. A signed acceptance +proves only the recorded repository check. It does not prove clean-machine +recovery, application startup, B2 negative permission tests, or a guaranteed +historical retention window. `protection_qualified` remains false. + +The small live workstation drill reconstructed and checked historical versions +both before and after the source key hid configuration and replaced a data +object. It restored the older snapshot with matching payload, hidden file and +symlink, using the encrypted kit for its password and a separate read-only +verifier identity. This is a simulated compromise on one workstation; it does +not establish off-host custody or replacement-machine recovery. See the +[qualification record](development/version-2.md#live-backblaze-b2-source-trial). + +### Central noncurrent-version retention (experimental) + +The experimental `cloud-retention` commands reclaim **noncurrent object +versions only**. They do not run Restic `forget` or `prune`, compact packs, or +delete current objects and delete markers. Keeping all current packs avoids +breaking deduplicated snapshots. Full repository compaction and live provider +qualification remain outside this implementation slice of +[issue #25](https://github.com/CruxExperts/BestBackup/issues/25). + +Run retention on the independently controlled management host with its own +deletion-capable credentials. Use an enrolled schema-3 repository binding whose +endpoint, bucket and prefix exactly match the private retention policy. The +shared repository operation ledger serializes deletion with local backup and +restore operations. Arrange a quiescent window across other hosts as well; +the local lease cannot exclude a writer on another machine. + +The mode-0600 policy remains schema 1 for generic S3. It contains `policy_id`, a +64-character random hex `catalog_id`, positive `revision`, exact `endpoint_url`, +`bucket`, nonempty `prefix`, full `repository_id`, trusted central verifier +primary fingerprint, `retention_counts` (`daily`, `weekly`, `monthly`), +`grace_seconds` (at least 2592000), `plan_max_age_seconds`, +`max_apply_objects`, `max_apply_bytes`, private absolute `state_dir`, nonempty +`known_good_manifest_digests`, and `pending_manifest_digests`. Keep central state +independently backed up. Losing an established catalog stops planning; +enrollment cannot silently recreate it. + +For Backblaze B2, schema 2 adds one exact native lock-metadata selector; every +other policy field remains required: + +```json +{ + "schema_version": 2, + "lock_metadata": { + "provider": "b2-native", + "bucket_id": "EXACT_BUCKET_ID" + } +} +``` + +This is only the B2-specific policy fragment. The bucket ID must match the +selected bucket and the policy's canonical HTTPS Backblaze S3 endpoint. Planning +rejects target or reader mismatches; versions evaluated for deletion require +exact identity and authorized protection metadata. Changed evidence stops apply, +and extra lock-metadata policy fields are rejected. Existing policy CLI arguments select the +schema; there are no new flags. A schema change requires an incremented policy +revision, new policy digest, matching grant, and new plan. Preserve the enrolled +catalog and original first-seen times; do not re-enroll to reset grace. + +The native metadata reader uses the same centrally configured standard B2 +application-key ID and secret as S3 requests. It does not accept a session +token. A broader central key is not required. Keep central retention and +provisioner/master credentials off source machines; never put them in the S3 +source process. The source key remains unchanged and restricted to source work. +Backblaze's [native `b2_get_file_info` response](https://www.backblaze.com/apidocs/b2-get-file-info) +exposes per-file retention and legal-hold authorization and values. The reader +accepts only authorized, explicit metadata: an authorized null retention mode +and null retain-until timestamp mean no retention, and the SDK maps an authorized +null legal-hold value to off. Missing or unauthorized metadata remains unknown. +The S3 `ObjectLockConfigurationNotFoundError` response is ambiguous for this B2 +binding and is not treated as proof that lock is absent. See Backblaze's +[native Object Lock API guide](https://www.backblaze.com/docs/cloud-storage-enable-object-lock-with-the-native-api). + +The separate mode-0600 recurring grant binds `schema_version: 1`, `policy_id`, +`catalog_id`, `policy_revision`, the exact file's `policy_sha256`, matching +target and repository fields, `action: "delete_noncurrent_versions"`, an +`expires_at` timestamp, and positive `max_apply_objects`/`max_apply_bytes` limits. +Its `stop_conditions` must be exactly `cancelled`, `input_drift`, +`lock_unknown_or_changed`, `mutation_outcome_uncertain`, and +`policy_or_grant_expired`. A broad management key does not expand this grant. + +Keep `records.json` private; its JSON array lists the absolute directories of +all central acceptance records for this catalog. Each directory contains the +signed acceptance **and** original signed `checkpoint.jsonl`/`checkpoint.sig`. +Signatures and exact dependencies are verified. Daily/weekly/monthly selection +uses central verification time; previously known records cannot be omitted. +Selected checkpoints, known-good and pending pins, and newly accepted checkpoints +within the grace period remain protected. Missing or wrong-size protected object +versions stop planning and application. + +```bash +bbackup production --config config.v3.json --bindings bindings.v3.json \ + cloud-retention catalog-enroll --repository cloud \ + --policy-file /secure/retention/policy.json --grant-file /secure/retention/grant.json +bbackup production --config config.v3.json --bindings bindings.v3.json \ + cloud-retention plan --repository cloud \ + --policy-file /secure/retention/policy.json --grant-file /secure/retention/grant.json \ + --records-file /secure/retention/records.json --quiescent-window-confirmed +``` + +Catalog enrollment is a one-time action. Inspect the plan's sanitized counts +and digests and private immutable file before applying its exact ID and SHA-256. +The noncurrent grace clock starts at central first observation, rather than the +upload timestamp. Enabled lifecycle expiration rules are refused. Versioning, +bucket locking, per-version retention and legal holds are checked conservatively. +The live verifier read succeeded after the separate central identity received +`readBucketLifecycleRules`; versioning was enabled and no lifecycle expiration +rules applied. No lifecycle-write capability was added. Separate canaries +confirmed denial of source retention/hold administration and exact deletion of +one unprotected version by the central retention identity. Separate fresh +governance-locked and legally held canaries each refused permanent deletion by +that identity without bypass capability. Independent readback confirmed survival, +and the native reader observed both protections. The provisioning identity +subsequently removed protection and deleted only those canaries. These checks do +not qualify aged-version reclamation or the promised historical recovery window. + +A separate two-object workstation probe used the restricted source key for S3 +deletion **without** a version ID. Both the governance-locked current object and +the legally held current object received one hide marker and became absent by +current-name lookup. Independent native inspection confirmed that each exact +original version and its protection survived. Read-only reconciliation found no +uncertain operations; the provisioning identity then removed protection and +deleted only those two canaries and their markers. This tests name hiding and +historical-version survival, not aged reclamation or a guaranteed recovery window. + +A live central-retention rehearsal enrolled a catalog, planned with +a fresh thirty-day first-seen grace period, ran a guarded zero-apply, and +reconciled an empty plan. The plan identified 10 protected versions among 31 +inventory versions. Before/after inventory matched and no S3 delete request was +sent. Empty reconciliation made zero provider reads, so it does not demonstrate +recovery from an uncertain deletion. The S3 lock-configuration response was an +ambiguous `ObjectLockConfigurationNotFoundError`. The native reader separately +verified the exact original configuration and data versions, including authorized +null lock settings, while the S3 lock flag was false. A subsequent production CLI +run completed setup, deliberate schema-3 enrollment, schema-2 planning, +zero-candidate apply and empty reconciliation using the existing catalog and +repository lease. It preserved established first-seen times, the original +policy/grant files and all 31 versions, including 10 protected versions. No aged +deletion or uncertain mutation was exercised, and no cost or savings were +measured. + +```bash +bbackup production --config config.v3.json --bindings bindings.v3.json \ + cloud-retention apply --repository cloud \ + --policy-file /secure/retention/policy.json --grant-file /secure/retention/grant.json \ + --records-file /secure/retention/records.json --plan-id EXACT_PLAN_ID \ + --plan-sha256 EXACT_PLAN_SHA256 --quiescent-window-confirmed +``` + +Application revalidates authority, expiry, dependencies, inventory and locks, +then journals intent before each exact-version delete. Mutation retries are +disabled. Cancellation stops new dispatch; an uncertain response blocks replay. +Inspect the target and reconcile the original journal using read-only requests: + +```bash +bbackup production --config config.v3.json --bindings bindings.v3.json \ + cloud-retention reconcile --repository cloud \ + --policy-file /secure/retention/policy.json \ + --plan-file /secure/retention/state/retention-plans/EXACT_PLAN_ID.json +``` + +Fixture tests include real signed recovery records, dependency loss, lock and +lifecycle changes, cancellation, contention, expiry, and lost responses before +and after deletion. They do not establish B2/AWS semantics or production savings. +`protection_qualified` remains false. + +### B2 lifecycle-overlap guard (experimental) + +On 2026-10-21, a new `allPrivate` B2 bucket was created with File Lock enabled, +default retention `NONE`, and overlapping native lifecycle rules: a broader +prefix hides files after 3,650 days, while a nested prefix deletes noncurrent +versions after 3,651 days. Native B2 allows overlapping prefixes and combines +the smallest non-null value for each action +([native lifecycle rules](https://www.backblaze.com/docs/en/cloud-storage-lifecycle-rules)). +The S3-compatible `Get Lifecycle Configuration` read returned the three +corresponding actions: timed hide, orphan-marker removal, and noncurrent-version +deletion. Backblaze documents this mapping in its +[S3 Get Lifecycle Configuration reference](https://www.backblaze.com/apidocs/s3-get-lifecycle-configuration). +Because S3 `Put Lifecycle Configuration` rejects overlapping prefix filters, +the rules were installed through Native B2 only, at bucket creation +([S3 Put Lifecycle Configuration](https://www.backblaze.com/apidocs/s3-put-lifecycle-configuration)). + +The one-hour bucket-scoped verifier key had only `listBuckets`, `readBuckets`, +`readBucketLifecycleRules`, and `readBucketRetentions`; it had no object or +administration capabilities. The nonroot verification container had a +read-only root and fixture mounts, a separate writable work mount, and no +Envman or provisioner credentials. Native readback confirmed bucket type, +File Lock, default retention, and both rules; S3 readback confirmed enabled +versioning and the mapped actions. The actual central-retention inspection +raised `CloudError: Enabled bucket lifecycle expiration blocks central retention`. +A read-only first-entry inventory observation found no versions; its receipt is +bounded to that first page. No objects were uploaded, no bucket setting was +updated, and no deletion was attempted. This tests the configuration guard only, +not elapsed expiry, physical reclamation, or an object-level protection window. +The [full qualification record](development/version-2.md#live-b2-lifecycle-overlap-guard-trial) +also records the separate maintenance-interruption result. + +### Central snapshot maintenance (experimental) + +The separate experimental `cloud-maintenance` commands retire obsolete visible +snapshots before historical-version reclamation. Run them on the central +management machine. They preserve every checkpoint still protected by the +central daily/weekly/monthly selection, known-good and pending pins, or acceptance +grace. They forget only explicit full IDs that are visible, independently accepted +and no longer protected. Unknown snapshots remain untouched. Full repository +manifests can conservatively pin data shared with older checkpoints. +The current production maintenance path requires a canonical Backblaze B2 S3 +endpoint; it does not qualify generic S3 or AWS maintenance. + +Maintenance runs Restic `forget` with the exact planned IDs, then +`prune --max-repack-size 0`. This avoids repacking shared packs, while still +allowing removal of wholly unused packs and index rewrites. B2 versionless +removal hides names; it does not permanently remove historical versions. +Permanent reclamation remains the separate `cloud-retention` workflow after +its real first-observed noncurrent grace. Never reset a catalog or backdate its +records to make a version eligible. + +A schema-3 central policy retains the existing target, retention and protection +fields, including the B2 native `lock_metadata` selector when used, and adds: + +```json +{ + "schema_version": 3, + "maintenance": { + "mode": "forget-and-prune-no-repack", + "resource_budget": { + "schema_version": 1, + "staging_max_bytes": 1048576, + "minimum_free_bytes": 65536, + "cache_max_bytes": 0 + }, + "timeout_seconds": 3600 + } +} +``` + +This is a policy fragment, not a complete policy. A schema-2 recurring grant +retains the existing authority fields and sets +`action: "maintain_current_repository"`. A maintenance grant cannot authorize +permanent-version deletion; a deletion grant cannot authorize maintenance. +Plans bind the exact policy, grant, accepted records, protected versions, current +inventory, Restic repository identity, visible snapshots and private bindings. +Current object inventory, excluding transient Restic `locks/` entries, is bounded +by the policy and grant limits, including retained or held current objects. +Those limits do not promise a maximum prune request count or memory use. + +Keep a separate private maintenance bindings file. It references a distinct +application key for the Restic child, rather than the central version-deletion +credential pair: + +```json +{ + "schema_version": 1, + "staging_dir": "/secure/maintenance-staging", + "access_key_id_env": "BBACKUP_B2_MAINTENANCE_KEY_ID", + "secret_access_key_env": "BBACKUP_B2_MAINTENANCE_KEY", + "credential_profile": "restricted" +} +``` + +Use an owned private, dedicated mounted filesystem within the staging budget; +cache remains disabled and free-space reserves are observed continuously. The +budget belongs to central policy and does not require a capture job on this +machine. The standard restricted key is bound to the exact bucket and machine +prefix with read/list/write permissions and no permanent deletion or management +capability. An explicitly selected `management` profile permits a broader key; +it does not expand the reviewed target or grant. Do not put inline secrets in +bindings or pass provisioning/deletion credentials to Restic. + +For a new catalog, enroll it once, then explicitly register independently signed +acceptance records with `cloud-maintenance records-enroll`. Registration uses +the existing catalog owner and preserves established acceptance and first-seen +state. Logical maintenance never silently accepts new records. Existing catalogs +must be preserved, including when changing policy schema or grant action. + +```bash +# New catalog only; skip this command for an established catalog. +bbackup production --config config.v3.json --bindings bindings.v3.json \ + cloud-maintenance catalog-enroll --repository cloud \ + --policy-file /secure/maintenance/policy.json \ + --grant-file /secure/maintenance/grant.json +bbackup production --config config.v3.json --bindings bindings.v3.json \ + cloud-maintenance records-enroll --repository cloud \ + --policy-file /secure/maintenance/policy.json \ + --grant-file /secure/maintenance/grant.json \ + --records-file /secure/maintenance/records.json +bbackup production --config config.v3.json --bindings bindings.v3.json \ + cloud-maintenance plan --repository cloud \ + --policy-file /secure/maintenance/policy.json \ + --grant-file /secure/maintenance/grant.json \ + --records-file /secure/maintenance/records.json \ + --maintenance-bindings-file /secure/maintenance/bindings.json \ + --quiescent-window-confirmed +``` + +Inspect the returned target IDs and immutable plan before applying its exact +`--plan-id` and `--plan-sha256` with `cloud-maintenance apply`; retain the same +policy, grant, records, bindings and quiet-window arguments. If the attempt is +uncertain, use `cloud-maintenance reconcile` with those exact arguments and the +original plan ID and digest. Do not run apply again to discover what happened. + +Plan and reconcile only while repository writers remain stopped for the entire +operation, with `--quiescent-window-confirmed`. Their Restic inspections use +`--no-lock --no-cache` and observe repository locks before and after reads. +These observations cannot prevent another host from starting a writer. Apply +uses the shared local repository lease and normal Restic mutation locks. + +Each mutation has a durable phase intent. An interrupted or uncertain attempt +blocks replay of the plan; inspect and reconcile the original attempt instead. +Healthy repository checks alone do not prove that prune completed. Reconciliation +also verifies protected historical versions and records only what its evidence +establishes. No automatic replay or implicit repository initialization occurs. + +A small live B2 workstation trial on 2026-10-21 completed the production CLI +maintenance phases, including native no-repack prune, with zero snapshots +selected for forgetting. Both independently accepted snapshots restored +correctly afterward, all 32 previously observed cloud versions survived, and +the catalog stayed unchanged. The maintenance phase used a 1 MiB staging mount +and disabled cache with separate scoped read-only and read/list/write keys. +Central checkpoint preparation used a separate workspace. This does not +qualify aged physical reclamation or the promised recovery window; see the +[maintenance trial and limits](development/version-2.md#live-b2-central-maintenance-trial). +In a separate live changed-plan test, an eight-byte probe invalidated a fresh +plan. Apply was rejected before maintenance-ledger entry, forget, or prune; +ordinary inspection still created and released two temporary repository locks. + +A separate live interruption trial used one zero-forget plan and one small +fixture. After confirming the owned native prune process, durable intent, and a +fresh lock witness, one SIGINT reached the owned CLI. The native process +stopped and the CLI exited 3. After Restic processes were quiescent, one +read-only reconciliation recorded an abandoned plan while the prune outcome remained uncertain; apply +was not replayed. A no-lock full read and two restores matched the fixture, all +69 previously observed provider objects and markers survived, and the catalog +was unchanged. No permanent version deletion was observed. The drill does not +claim prune completed; see the [interruption trial limits](development/version-2.md#live-b2-central-maintenance-interruption-trial). + ## Bind the cloud repository Follow [the quick start](../QUICKSTART.md) to create your local repository, source, @@ -109,6 +514,133 @@ succeeded; reconcile uncertain operations before retrying. ## Retries, backoff, and interruption +### Remote primary with bounded staging (experimental) + +An experimental `remote-primary` repository can capture ordinary files directly +through Restic into an HTTPS S3 endpoint. It does not create a second local +repository or archive. Local-first capture and replication remain available. +This implementation has local regression evidence and a small live B2 +workstation source trial, recorded in the [version 2 development qualification +record](development/version-2.md#live-backblaze-b2-source-trial). Final +remote-server and provider-protection qualification remain deferred. + +For example, the portable repository and job declarations are: + +```json +{ + "repositories": [{"name": "cloud", "role": "remote-primary"}], + "jobs": [{ + "name": "daily", "repository": "cloud", "sources": ["files"], + "resource_budget": { + "schema_version": 1, "staging_max_bytes": 67108864, + "minimum_free_bytes": 8388608, "cache_max_bytes": 0 + } + }] +} +``` + +These are fragments of a schema-3 policy, not a complete configuration. Bind the +repository to its HTTPS S3 URL and private password file as above, and set +`staging_dir` in private host bindings. The staging directory must be an owned, +mode-0700, dedicated mounted filesystem whose entire capacity is at most the +configured staging budget. It must contain only the stable bbackup staging lock +before a new capture starts. Read-only reconciliation can inspect validated +retained exports on that mount. Arrange the mount separately, including persistence +for scheduled jobs; bbackup neither mounts it nor deletes unrelated content. +Choose the limits from actual available capacity and export requirements. + +This first profile requires `cache_max_bytes: 0` and uses Restic `--no-cache`. +Explicit `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` credentials must be in +the supervised process environment. Inherited Restic options and AWS profiles +cannot redirect the operation. The default schedule remains `daily` (24 hours). +Database sources retain consistent native exports or the SQLite backup API; +their staging writes are checked before writing, and unsupported combinations +fail rather than fall back to copying a live database file. Native Docker and +manual or scheduled replica copies involving a remote primary are currently +refused because they lack the same bounded export/copy support. + +A staging-wide nonblocking lease serializes users of that volume. The runner +checks disk pressure even while a child is silent; it terminates a child on a +pressure failure and preserves uncertain-operation reconciliation. Results +include `primary_destination`, separate local/cloud completion, and resource +observations. Failures use `resource_budget_exceeded`. The dedicated filesystem +provides a hard staging-capacity ceiling. Free-space reserve checks are observed +thresholds, not a guarantee against unrelated writers between checks; results +report `free_space_reserve_hard_guaranteed: false`. Observed changes in filesystem +use are not attributed exclusively to the backup process. + +A small local emulator trial captured 2,245,463 logical bytes with only +1,769,472 bytes of combined writable filesystem capacity. A subsequent +controlled pressure failure stopped capture, and a separate check and full +restore verified the earlier recovery point. This used compressible synthetic +data and establishes no production capacity estimate; see the +[aggregate-capacity and pressure trial](development/version-2.md#aggregate-capacity-and-active-pressure-trial). + +Completed remote database exports are unique recovery artifacts, not disposable +cache. Their operation-owned staging files and private hash records survive a +failed or uncertain upload, cancellation, or receipt failure. New captures stay +blocked while exports remain. Successful reconciliation checks the exact remote +capture tag and manifest before repairing a receipt and releasing the matching +stage; it never repeats the upload. A reviewed failed outcome retains the data. +Before reconciling a remote capture, stop repository writers for the entire +inspection and explicitly pass `--quiescent-window-confirmed` alongside +`--repository-reviewed`. Inspection uses Restic's `--no-lock` to avoid writing +cloud lock objects and refuses observed repository locks before and after its +reads. These observations cannot prevent another machine from starting a writer; +the operator must maintain the quiet window until reconciliation returns. + +Discover retained capture IDs and current artifact-record digests, including +captures stopped before dispatch: + +```bash +bbackup production --config config.v3.json --bindings bindings.v3.json \ + runs retained-exports --repository cloud --limit 20 --offset 0 +``` + +The counts are declared metadata, not file-hash verification. After independently +recovering the needed export or deliberately accepting its loss, record the +reviewed failed outcome through `runs reconcile` and inspect the updated record +digest. Explicit disposal is then bound to that exact record: + +```bash +bbackup production --config config.v3.json --bindings bindings.v3.json \ + runs discard-exports --repository cloud --operation-id EXACT_OPERATION_ID \ + --artifact-record-sha256 EXACT_CURRENT_RECORD_SHA256 +``` + +The command refuses stale records or foreign files and keeps its disposal audit. +Do not remove staging files or edit private records by hand. + +For survival across host reboot or power loss, use persistent dedicated staging. +On `tmpfs`, exports survive process exit but disappear with the host's memory; +the persistent record cannot recreate them, and reconciliation fails closed. + +### Credential roles + +Least-privilege source credentials are the standard production profile: read, +list, and new writes within one machine's encrypted repository, with historical +backup versions protected against permanent deletion or alteration. Management +credentials are also supported when explicitly supplied for provisioning, +verification, retention, or recovery. Keep those roles and their process +environments separate. A management key's failure to satisfy the restricted +source profile does not make it unusable for its intended management operation. +Supplying a broad key never expands the reviewed target or retention grant. + +For Backblaze's S3 route, use a standard application key with the capabilities +needed for that role. A master key can remain in the native-API provisioning +environment; do not transfer it into S3 credential variables. Provisioning an +application key requires `writeKeys`. See Backblaze's +[application-key roles](https://www.backblaze.com/docs/cloud-storage-application-keys) +and [key-creation API](https://www.backblaze.com/apidocs/b2-create-key). + +For local development, distinct names such as `BBACKUP_B2_PROVISIONER_KEY_ID` +and `BBACKUP_B2_PROVISIONER_KEY` avoid collisions with source credentials. Transfer +only the intended role's values into the provider's standard environment names +inside that operation's private process. Never put provisioning keys on a source +host by default. B2 `writeFiles` permits hiding names and uploading replacement +versions; strict append-only current names are not claimed. Tests must prove +that protected historical recovery points remain unchanged and recoverable. + Native B2 inspection uses pinned Backblaze SDK retries and runs in a supervised child process with a 120-second deadline. Cancellation stops and reaps the process group, including SDK backoff waits. Credentials remain in memory. @@ -129,11 +661,13 @@ operation. Follow the [operation reconciliation guide](development/version-2.md# ## Storage support and recovery protection A working destination establishes where a backup can be stored, not production -qualification. B2 and AWS remain outside the selected stable-release support -boundary until live provider capture/check/restore drills are recorded. Thirty-day -recovery after host compromise additionally depends on credentials, version -retention, lifecycle overlaps, an independent signed checkpoint, and an actual -historical-version restore drill; each remains deferred. +qualification. The limited B2 source-path drill is recorded, but B2 and AWS +remain outside the selected stable-release support boundary. AWS still has no +live capture/check/restore result. Thirty-day recovery after host compromise +additionally depends on credentials, version retention, lifecycle overlaps, an +independent signed checkpoint, and an actual historical-version restore drill. +The workstation historical-version restore passed, while a guaranteed thirty-day +window, independent custody and replacement-machine recovery remain unqualified. Keep current repository objects indefinitely: old deduplicated packs can still be required by recent snapshots. Retain hidden or overwritten versions for at diff --git a/docs/development/release-dependencies.json b/docs/development/release-dependencies.json index 17853da..175a33a 100644 --- a/docs/development/release-dependencies.json +++ b/docs/development/release-dependencies.json @@ -16,72 +16,74 @@ ], "package_version_coverage_verified_against_lockfile": true, "query_failures": 0, - "raw_evidence_sha256": "3ff337b3de5408e562e5f14a3f366db3010ea740e6d90f4ec8077df65be1f0fc", + "raw_evidence_sha256": "1c04489e2becf51c2ed9d2acea18bfbbc2db210c9f5a2d331c53ec7263eaf3b8", "result": "No advisory records were returned for these exact package versions by OSV, GitHub Advisory Database, deps.dev, or PyPI at retrieval time.", - "retrieved_at_utc": "2026-10-19T09:40:08.281608+00:00", + "retrieved_at_utc": "2026-10-21T09:12:55.211479+00:00", "scope": "All 51 exact PyPI registry package versions in uv.lock, plus the exact setuptools and wheel versions resolved for the build-system requirements.", "sources": [ { "advisory_hits": 0, "endpoint": "https://api.osv.dev/v1/querybatch", "http_status_counts": { - "200": 1 + "200": 2 }, "independence_class": "A", "name": "OSV", "package_version_queries": 53, - "requests": 1, - "retrieved_at_utc": "2026-10-19T09:39:51.482880+00:00" + "requests": 2, + "retrieved_at_utc": "2026-10-21T09:12:55.211479+00:00" }, { "advisory_hits": 0, "endpoint": "https://api.github.com/advisories?ecosystem=pip&affects=PACKAGE%40VERSION", "http_status_counts": { - "200": 53 + "200": 54 }, "independence_class": "B", "name": "GitHub Advisory Database", "package_version_queries": 53, - "retrieval_window_utc": [ - "2026-10-19T09:39:52.016798+00:00", - "2026-10-19T09:40:05.733216+00:00" - ] + "retrieved_at_utc": "2026-10-21T09:12:55.211479+00:00", + "requests": 54 }, { "advisory_hits": 0, "endpoint": "https://api.deps.dev/v3/systems/pypi/packages/PACKAGE/versions/VERSION", "http_status_counts": { - "200": 53 + "200": 54 }, "independence_class": "E", "name": "deps.dev", "package_version_queries": 53, - "retrieval_window_utc": [ - "2026-10-19T09:39:52.293682+00:00", - "2026-10-19T09:40:06.436922+00:00" - ] + "retrieved_at_utc": "2026-10-21T09:12:55.211479+00:00", + "requests": 54 }, { "advisory_hits": 0, "endpoint": "https://pypi.org/pypi/PACKAGE/VERSION/json", "http_status_counts": { - "200": 53 + "200": 54 }, "independence_class": "E", "name": "PyPI JSON API", "package_version_queries": 53, - "retrieval_window_utc": [ - "2026-10-19T09:39:52.508368+00:00", - "2026-10-19T09:40:08.279763+00:00" - ] + "retrieved_at_utc": "2026-10-21T09:12:55.211479+00:00", + "requests": 54 } - ] + ], + "remediation": { + "advisory": "GHSA-6q48-6c7w-9gw2", + "package": "certifi", + "previous_version": "2026.5.20", + "resolved_version": "2026.10.19", + "source": "https://github.com/advisories/GHSA-6q48-6c7w-9gw2", + "note": "The initial lookup identified the affected version; the fixed version was separately queried across the same four sources. Other exact package versions are unchanged." + } }, "lockfile": { - "local_project_version": "2.0.3", + "local_project_version": "2.0.4", "locked_pypi_packages": 51, "path": "uv.lock", - "sha256": "7cf0bfc2b287378e4fe893a2f2ffc20379d165ce55fe6a518bc749f5a701c9fb" + "sha256": "d1163858fc14e6aa811e96ca6ae2136def3482b681bfe08434254b112203d6e9" }, "package_versions": [ { @@ -106,7 +108,7 @@ }, { "name": "certifi", - "version": "2026.5.20" + "version": "2026.10.19" }, { "name": "cffi", @@ -298,8 +300,48 @@ } ], "release": { - "release_date_utc": "2026-10-19", - "version": "2.0.3" + "release_date_utc": "2026-10-21", + "version": "2.0.4" }, - "schema_version": 1 + "schema_version": 1, + "external_build_tools": { + "uv": { + "version": "0.13.4", + "source": "https://pypi.org/project/uv/0.13.4/", + "usage": "Hashed bootstrap for CI and the four supported Ubuntu release-build targets.", + "artifacts": [ + { + "platform": "linux-amd64", + "filename": "uv-0.13.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sha256": "05e4332791b4d55b6dde8695b7fa6079b26991519f44760887c5dfbd5ca4fe53", + "yanked": false + }, + { + "platform": "linux-arm64", + "filename": "uv-0.13.4-py3-none-manylinux_2_28_aarch64.whl", + "sha256": "7b3cdf868e5415631fe797b648c0d6945d38d388f23367bdd0b6332e80a1fcb0", + "yanked": false + } + ], + "security_review": { + "advisory": "GHSA-hq85-5cpj-mr62", + "result": "No advisory records returned for the exact uv 0.13.4 version by OSV, GitHub Advisory Database, deps.dev, or the PyPI JSON API.", + "exact_version_only": true, + "raw_evidence_sha256": "d5f443900b224786589136f8447b405d25142b2355790f5bb97e0c3ec1082135" + } + }, + "restic": { + "version": "0.19.1", + "role": "Pinned executable supplied to release package builds and used by bbackup backup/restore operations.", + "advisory": { + "id": "GO-2026-5932", + "source": "https://pkg.go.dev/vuln/GO-2026-5932", + "scope": "The advisory concerns the separate self-update OpenPGP verification path present in restic 0.19.1.", + "bbackup_usage": "Packaging does not invoke restic self-update; release archives are version-pinned and SHA-256 checked, and the package builder verifies supplied executable bytes.", + "limitation": "This records the unused self-update-path caveat only and makes no broader claim about backup or restore behavior.", + "versioned_source": "https://github.com/restic/restic/blob/v0.19.1/internal/selfupdate/verify.go", + "upstream_fix_pull_request": "https://github.com/restic/restic/pull/21997" + } + } + } } diff --git a/docs/development/version-2.md b/docs/development/version-2.md index 656ebf7..f92da3e 100644 --- a/docs/development/version-2.md +++ b/docs/development/version-2.md @@ -1,12 +1,39 @@ # Version 2 development -> Stable v2.0.2 production qualification is limited to the tested scope -> recorded below. Other implemented paths remain unqualified or deferred. -> See the 2026-10-17 qualification record for exact evidence and limits. +> Stable v2.0.4 production qualification retains the tested core scope recorded +> below. Experimental additions remain unqualified. The 2026-10-17 record keeps +> the original v2.0.3 evidence and limits. + +## v2.0.4 release scope and known limits + +The v2.0.4 stable qualified scope is unchanged from v2.0.3: local file capture +and restore; live capture and restore for PostgreSQL 17.11, MySQL 8.4.12, and +MariaDB 11.8.9; headless dashboard happy paths; and AMD64 Ubuntu 24.04/26.04 +packages. The separate user-timer reboot qualification applies only to the +components and conditions recorded in its dated evidence below. + +This release also ships experimental restricted B2 source-key assessment and +central recovery acceptance, central maintenance and noncurrent-version +retention, HTTPS S3 remote-primary capture with bounded staging, and opt-in +reviewed Linux file and SQLite coverage with a manual PostgreSQL covered-capture +path. These additions are implemented workflows, not new production support. +They do not establish B2 or AWS protection, a guaranteed thirty-day recovery +window, off-host custody, final remote-server qualification, complete-machine +recovery, or general automated PostgreSQL coverage. `protection_qualified` +remains false. Requests #23–#26 remain open; this release does not close them. + +On 2026-10-21, the v2.0.4 candidate passed the full local suite with 1,242 +passed tests and one skipped test. The exact wheel and source archive built +and passed isolated installed-artifact smoke tests. Ruff, Python compilation, +version alignment, generated CLI catalogs, Markdown standards, publishing +readiness, and artwork provenance checks passed. The dependency audit found +and corrected the affected `certifi` pin to 2026.10.19; exact-version lookups +for the corrected dependency returned no advisory records at retrieval time. +These checks do not expand the production qualification boundary above. ## Direction and delivery order -The selected stable v2.0.2 release scope is local/file capture and restore, +The selected stable v2.0.4 release scope is local/file capture and restore, live capture-restore drills for PostgreSQL 17.11, MySQL 8.4.12, and MariaDB 11.8.9, headless dashboard happy paths, and AMD64 Ubuntu 24.04/26.04 packages. The CLI and Textual dashboard share one operations service. Existing 1.x commands @@ -25,6 +52,715 @@ An implemented capability does not establish completion of its entire delivery s ## Implemented capabilities +### Experimental cloud and machine recovery features + +Open requests are sequenced as restricted B2 recovery +([#23](https://github.com/CruxExperts/BestBackup/issues/23)), central retention +([#25](https://github.com/CruxExperts/BestBackup/issues/25)), remote-primary disk +budgets ([#26](https://github.com/CruxExperts/BestBackup/issues/26)), and integrated +privileged Linux recovery ([#24](https://github.com/CruxExperts/BestBackup/issues/24)). +Linux coverage discovery can proceed independently. Each request needs its own +implementation, focused regressions, documentation, review, signed commit, and +acceptance evidence before closeout. Stable qualification and issue closeout +require completion and final source/artifact validation; this release ships the +current workflows experimentally without closing the requests. A mocked +provider result cannot qualify B2. All four requests remain open; the live +trials below do not close them. + +The experimental B2 slice adds a restricted source-key metadata assessment to +native B2 inspection and an independent `recovery accept` command. Acceptance +reconstructs signed exact versions using the escrow kit, checks the repository +and snapshot, and signs a central record with a separate verifier fingerprint. +It does not establish independently controlled key custody by itself. + +A small live B2 source-path trial and seven denied-operation checks passed on +2026-10-20; details and limits are recorded below. Historical-version recovery +after simulated source compromise also passed on this workstation. A separate +live lifecycle-overlap guard trial verifies only the configured guard and its +central-retention refusal; elapsed lifecycle expiry, physical reclamation, the +thirty-day grace period, measured transfer costs, and independently controlled +off-host escrow remain outstanding. No live AWS result is recorded. +Complete privileged Linux capture and replacement-machine recovery also remain +unqualified. Do not close these issues or expand the supported release scope +from fixture tests. + +### Reviewed Linux coverage (experimental) + +An opt-in schema-3 job can declare `coverage_policy` and map its job name to a +private `coverage_declarations` path in host bindings. Ordinary jobs keep their +existing source-selection behavior. Coverage discovery records bounded Linux +account/group, mount, package, application-unit and selected-tree metadata in +private state. Public results contain counts, digests and reason codes only. +`coverage_status: complete` means the exact declared scope is accounted for; +it is not a claim that every machine file or application is recoverable. + +Restic service operations use an explicit cache-free fallback when their child +environment provides neither `HOME` nor `XDG_CACHE_HOME` and no cache option was +selected. This supports verified restore from minimal system-service +environments without relying on an interactive login environment. + +For example, this is a portable job-field fragment: + +```json +{ + "coverage_policy": { + "schema_version": 1, + "approved_roots": ["system", "homes"], + "require_privileged": true, + "filesystem_boundaries": "explicit", + "metadata_expectations": { + "numeric_owner": true, "mode": true, "acl": true, + "xattr": true, "links": true + } + } +} +``` + +The separate mode-0600 declaration is schema 1. `roots` maps each approved root +ID to an absolute `path`, configured `source_id` and `adapter`. It also requires +`exclusions`, `mount_decisions`, `identity_decisions`, `applications`, +`package_reconstruction`, and `reconstruction_evidence`. Empty arrays are +permitted, but unresolved inventory gaps block approval. See the maintained +[fixture declarations](../../tests/test_linux_coverage.py) for the exact shapes. +Covered capture supports ordinary files and individually transactional SQLite +exports. Each SQLite root requires an application declaration whose state root +is that exact database path and whose reconstruction procedure has verified +private evidence. An experimental manual PostgreSQL extension is described +below. MySQL/MariaDB and Docker remain unsupported for covered jobs; ordinary +jobs retain their existing adapters. Existing schema-1 private declarations +and the portable schema-1 coverage policy keep their current shapes. The +PostgreSQL extension uses a schema-2 private declaration and does not change +ordinary native PostgreSQL capture or restore. + +Application declarations use service-unit stems without the `.service` suffix. +Ordinary names, templates and instances such as `serial-getty@ttyS0`, including +valid systemd hex escapes, are supported. Names remain literal inventory keys; +malformed escapes, path separators and overlength names fail discovery. + +SQLite review binds the database's device/inode, numeric owner, mode, link count +and bounded extended attributes, including filesystem ACL attributes when +present. The database must be a readable regular file without symlink ancestors +or additional hard links. Replacement or metadata changes require fresh review; +ordinary database writes and journal-file churn do not. Unsafe journal links and +user exclusions covering the database prevent approval. + +Capture uses the existing SQLite backup API and omits the live database and its +`-wal`, `-shm` and `-journal` files from file-tree capture. The encrypted capture +manifest binds the export hash to the reviewed root, source identity, original +metadata, reconstruction recipe and baseline. Generic restore returns that +export and manifest; the reviewed reconstruction procedure must place the +database and restore its original metadata. This does not provide automatic +in-place application recovery or a shared transaction across files or databases. + +#### Experimental PostgreSQL covered capture + +The private schema-2 declaration adds `database_roots` for selected logical +databases and one `postgresql_clusters` entry. Each captured database maps to +the exact portable PostgreSQL source and private connection alias; the alias +and source selector must agree with the private host bindings. Here is only the +logical-root fragment, not a complete declaration; see the maintained +[PostgreSQL coverage fixture](../../tests/test_postgresql_coverage.py) for the +full valid shape: + +```json +{ + "database_roots": [ + {"id": "application_db", "adapter": "postgresql", + "source_id": "application_source", "cluster_id": "primary", + "connection": "private_connection_alias", "database": "application"} + ] +} +``` + +The declaration accounts for every database in the selected cluster, +including templates and databases that reject connections. Each gets an +explicit `capture` or `reconstruct` decision tied to its current name and OID; +nonconnectable databases must be reconstructed. A reconstructed database needs +actual procedure-execution and result-verification evidence bound to the +reviewed database and cluster inventory. The local administrative connection +must have the native inspection privileges needed to read cluster, database, +role, membership, and configuration metadata. `pg_controldata` must be +available on `PATH` at the matching major version. Its control-file system ID +must equal the SQL-reported ID, and SQL `data_directory` must exactly match the +declared absolute local path in the same filesystem namespace. Review also +binds that directory's device, inode, numeric owner and group, and mode, plus digests for +the private connection and password files, database and global inventories, +roles, authentication files, and configuration. + +Only one local primary cluster is supported. `pg_wal` must be inside the same +filesystem as the declared data directory; non-default tablespaces, external +WAL, external configuration or authentication includes, authentication +`@file` references, SSL-enabled arrangements, standbys, and unsupported +extensions, languages, or foreign objects are rejected. This is a bounded +local path, not general PostgreSQL cluster backup support. + +The administrator must coordinate a quiet window for role, database, +configuration, and schema changes throughout discovery, review, checks, and +capture. Normal row writes may continue. The boolean confirmation defaults to +false and does not stop external changes; scheduled covered PostgreSQL jobs +remain blocked because their timer dispatch cannot assert that window. Set the +flag only while that external coordination is active. A capture uses one +transactional native `pg_dump` custom archive per selected database. It does +not create one shared transaction across databases or between database and +file roots. + +After database accounting and restoration evidence pass, the literal reviewed +`data_directory` (including its live raw files and `pg_wal`) is excluded from +file-tree capture. Recovery procedure, evidence, and actual recovery-material +files must be private regular files inside the same job's approved file roots, +outside the data directory and declared exclusions. Individual file roots are +allowed. Their bytes are included in the encrypted snapshot. Its encrypted +manifest binds the reviewed approval, each database export hash, and each +recovery-artifact hash. Covered PostgreSQL restore verifies the archive and +all referenced recovery-material bytes from that snapshot before its first +target-database mutation. Existing trusted-archive acknowledgement and new +target database safeguards still apply. The source connection alias is +provenance only: restore uses a fresh destination-host connection alias and +does not require the original source binding. + +Captured databases also require a reviewed reconstruction procedure for +database-level metadata. Native restore creates a new database with the +captured encoding and locale, then runs `pg_restore`; it does not restore the +database owner, database ACL, or connection limit. The recovered procedure +must apply and verify those settings for the chosen destination name, +including PUBLIC revocations and grant options, and account for other reviewed +database attributes. Its execution evidence must +compare the reviewed source metadata with a reconstructed database before +coverage review and capture. Repeat that procedure from the recovered materials +after the final restore. The native `database_validated` result means that +`SELECT 1` succeeded after archive restore; `application_validated` remains +false. Neither result establishes completion of the reconstruction procedure. + +Use the manual commands only during the coordinated window; replace the +candidate digest, snapshot ID, and evidence path with the reviewed values: + +```bash +bbackup production --config config.v3.json --bindings bindings.v3.json \ + coverage discover --name application-stack --quiet-administrative-window-confirmed +bbackup production --config config.v3.json --bindings bindings.v3.json \ + coverage review --name application-stack --candidate-digest FULL_CANDIDATE_SHA256 \ + --quiet-administrative-window-confirmed +bbackup production --config config.v3.json --bindings bindings.v3.json \ + coverage check --name application-stack --quiet-administrative-window-confirmed +bbackup production --config config.v3.json --bindings bindings.v3.json \ + coverage recovery-evidence --name application-stack --snapshot FULL_SNAPSHOT_ID \ + --evidence-file PRIVATE_EVIDENCE_FILE --quiet-administrative-window-confirmed +bbackup production --config config.v3.json --bindings bindings.v3.json \ + jobs run --name application-stack --quiet-administrative-window-confirmed +``` + +On 2026-10-21, a local PostgreSQL 17.11 rehearsal passed in two isolated +containers with distinct machine identities, data directories, and recovery +workspaces. The source passed native enrollment, coverage discovery, review, +readiness checks, and manual capture. Its final encrypted snapshot contained +16 files totaling 10,006 logical bytes, including a 2,307-byte database archive +and five digest-bound recovery files; no raw PostgreSQL data-directory nodes +were present. A full repository read-data check passed over four snapshots +and eight packs. Adding an unaccounted database separately blocked coverage +and manual capture while preserving the accepted snapshot. + +The replacement restored from a copy of the encrypted repository and separate +test password escrow, using a fresh connection alias and a previously absent, +differently named target database. The native restore verified all five recovery +file hashes and completed in 6.82 seconds; this measures that command alone, +not application reconstruction or a production recovery deadline. The corrected +reconstruction procedure was executed and verified before final coverage review +and capture, then recovered from that snapshot and executed again. Separate SQL +comparisons matched database ownership, all three ACL entries, connection limit +7, table ownership and reader permissions, and three committed UTF-8 rows. +An application-role query returned all three rows. The CLI still reported +`application_validated: false`; these separate fixture checks supply the +application evidence. + +This is local emulation of the bounded manual path. It does not qualify +off-host custody, provider access, whole-machine recovery, or unattended +PostgreSQL coverage. The path ships experimentally in v2.0.4 and adds no +qualification to the v2.0.3 core scope; native PostgreSQL support for ordinary +jobs is unchanged. + +The full regression run reported 1,241 passed, one known Docker skip, and one +dashboard assertion failure. That assertion was updated to expect the new +quiet-window field's false default; all seven dashboard tests then passed. +Runtime code was unchanged between those runs. Version, publishing-readiness, +generated-interface, compilation, Markdown, lint, and diff checks passed. + +Each exclusion or reconstruction decision names a reason, consequence, recipe +ID and procedure SHA-256. A referenced private procedure and private evidence +record must match their digests and identify verified local execution. These +records account for local scope decisions; they do not independently establish +machine recovery. Symlinked homes or application state outside approved scope, +unreadable data, unaccounted users/services/mounts and missing reconstruction +evidence leave coverage incomplete. Declared exclusions become literal Restic +exclusion patterns during capture, including paths containing glob characters. + +```bash +bbackup production --config config.v3.json --bindings bindings.v3.json \ + coverage discover --name system +bbackup production --config config.v3.json --bindings bindings.v3.json \ + coverage review --name system --candidate-digest EXACT_CANDIDATE_SHA256 +bbackup production --config config.v3.json --bindings bindings.v3.json \ + coverage check --name system +bbackup production --config config.v3.json --bindings bindings.v3.json \ + jobs run --name system +``` + +Discovery never approves coverage. Review binds an exact complete candidate to +the enrolled machine, user, configuration, bindings, declaration and scope. +Capture checks the current baseline again and refuses changed mappings or +structural drift. Coverage receipts are separate per declaration/scope. Secrets +inside an explicitly reviewed file root may enter the encrypted snapshot; +runtime state, repositories and staging remain excluded from source overlap. + +`service render-system` renders root service/timer units without installing +them or elevating privileges. It requires root identity and root-controlled +executable, configuration, bindings and path ancestors. The system timer is +persistent and the service dispatches through the same production command. +Review and install those units deliberately using the system administrator's +normal procedure. User-level `service install` remains unchanged. + +Units for opt-in covered jobs use `PrivateTmp=no` so scheduled discovery and +capture see the same reviewed `/tmp` and `/var/tmp` sources. Ordinary jobs retain +`PrivateTmp=yes`; their existing hardening is unchanged. Coverage still refuses +runtime-state/repository recursion and detects real scoped mount changes. + +`coverage recovery-evidence` assesses a private record against the exact current +baseline and snapshot. Local fixture records can produce `recovery_status: +emulated`; absent, incomplete or unverified independent records stay `untested`. +No command marks complete-machine recovery qualified. Current tests include +small real Restic capture/restore with enforced exclusions, account and service +drift, private evidence integrity and unprivileged readability. A local WAL-mode +SQLite/files drill restores committed rows, omits uncommitted rows and raw +database/journal files, validates export provenance, and reconstructs original +metadata with the reviewed recipe. Privileged +complete-machine recovery and final remote-server qualification remain +outstanding for #24. + +On 2026-10-21, the opt-in `covered-root` scenario in +[`qualify_systemd.py`](../../scripts/qualify_systemd.py) passed a separate +workstation rehearsal in two fresh Ubuntu 24.04 AMD64 guests. Both installed +the exact wheel built from signed commit `3755ee0`; its SHA-256 was +`7106094a242975cc51365d45c72f5518fb76eb2a0feadaee3a4211edd85748da`. +The source used actual Linux inventory, root-owned enrollment and rendered +system units, then captured through the root timer and performed a verified +production restore. The replacement used a distinct machine identity and OS +overlay; it received the dedicated recovery disk and candidate inputs, without +access to the source OS overlay or a plaintext application-data fallback. + +All twelve evidence checks passed: committed-only SQLite export and manifest +hashes, raw database/sidecar exclusion, restored synthetic secret and application +output, reconstructed fixture account, and numeric ownership, mode, hard/symbolic +links, extended attributes and an actual named-user ACL. Reconstruction executed +the reviewed procedures and started the restored application through systemd; +the fixture and backup units were verified afterward. The logical fixture contained +56,293 bytes on a 256 MiB recovery disk. The two-guest run took 90.1 seconds; +replacement completion appeared 18.5 seconds after its kernel boot, including +5.8 seconds in its qualification stage. These small-fixture measurements do not +estimate production recovery time. The harness's 39 focused tests passed. + +The result is `local-emulation-only`. Shared local recovery media does not +establish independently controlled off-host escrow, provider protection, or +complete-machine recovery. It does not extend the published v2.0.3 support +scope or complete the final remote-server acceptance. + +### Remote-primary and central retention (experimental) + +The experimental remote-primary implementation adds HTTPS S3 primary repositories, +versioned resource budgets, a dedicated capacity-bounded staging mount, disabled +cache, a shared staging lease, export write checks, and continuous runner +pressure checks. Completion results distinguish local and remote captures; +status exposes remote-primary capture separately from local capture. Free-space +reserves remain observed thresholds, and unsupported Docker/replica combinations +are refused. These implementation checks do not qualify B2 or whole-machine +recovery. A small live B2 workstation source trial is recorded below; the final +remote-server drill and production recovery qualification remain deferred. + +Remote database captures preserve completed exports in operation-owned staging +with private size/hash records until a successful receipt or verified read-only +reconciliation permits cleanup. Failed or uncertain uploads, receipt failures +and later-export failures retain completed artifacts and block new captures. +`runs retained-exports` exposes bounded IDs, declared counts and record digests, +including pre-dispatch captures without a ledger row; `runs list` remains a +ledger event stream. `runs discard-exports` requires the exact current record +digest and a persisted reviewed failed outcome, verifies owned files under the +staging lease, and keeps a disposal audit. It does not upload or replay a capture. +Persistent dedicated staging is required for reboot survival; `tmpfs` retains +exports only across process exit. Missing or changed retained data fails closed. +Remote capture reconciliation requires an explicitly acknowledged quiet window +and uses Restic `--no-lock` reads to avoid creating cloud lock objects. Lock +observations before and after inspection reject active locks but cannot enforce +quiescence across machines; repository writers must remain stopped throughout. + +The experimental central retention slice adds explicit catalog enrollment and +private recurring grants, signed central acceptance selection, exact manifest +dependency preservation, central observation-based noncurrent grace, conservative +lock/lifecycle checks, and single-request journaled deletion with read-only +reconciliation. It uses the enrolled repository's shared lease. Missing protected +versions or catalog records stop deletion. A schema-2 B2 policy can select a +native lock-metadata reader; schema-1 generic S3 policy behavior is unchanged. +This reclaims only noncurrent object versions; aged B2 reclamation, AWS behavior +and production savings remain outstanding for #25. Focused tests +include actual GPG-signed acceptance records and uncertainty before/after deletion. + +The separate experimental `cloud-maintenance` workflow uses the same central +catalog and independently accepted records to retire only visible accepted +snapshots that are no longer protected. It leaves unknown snapshots untouched, +uses a separate explicit Restic credential profile, and runs no-repack prune +before any later historical-version reclamation. Policy schema 3 and action-bound +schema-2 grants bind resource limits and immutable plans; explicit record +enrollment preserves established acceptance and first-seen state. Read-only +inspection requires operator-maintained quiescence and uses `--no-lock`, while +mutations keep native locks and journal intent without automatic replay. +Healthy reconciliation alone does not certify completed prune. Repacking, +aged live deletion and the promised recovery window remain unqualified. +See the [central maintenance workflow](../cloud-storage.md#central-snapshot-maintenance-experimental). + +Local validation includes a real three-snapshot Restic repository: the exact +obsolete accepted ID was forgotten, no-repack prune completed, and both the +protected and unknown snapshots restored afterward. That drill uses a test +double for the central cloud session; it is not live B2 maintenance evidence. +At maintenance-slice acceptance, the runtime suite passed 1,158 tests with two +skips. The Unix-socket case skipped +because the test temporary path exceeded the platform limit, then passed in a +separate short-path run. The legacy Docker roundtrip still skipped after its +backup returned false and supplies no Docker recovery qualification. + +#### Live B2 central-maintenance trial + +On 2026-10-21, development code at `d33354a` passed a live B2 +workstation trial using the previously measured two-snapshot fixture. Both +snapshots received genuine, separately signed central acceptance records. +The reviewed immutable plan protected both snapshots and their ten dependency +versions, covering 201,834 current payload bytes, and selected no snapshots to +forget. Production CLI execution completed all six phases, including an actual +`prune --max-repack-size 0`. A zero-forget plan still performs native mutations. + +A subsequent full read-data check and restores of both snapshots passed. +Payload, hidden-file, symlink, and initial/incremental contents matched the +fixture. All 32 versions observed before maintenance remained present, and the +central catalog digest was unchanged. The final inventory added 16 lock-object +versions and 16 hide markers, with no new index versions; permanent version +deletion was not observed. These changes include maintenance and its subsequent +verification, rather than measuring prune alone. + +The isolated central process used distinct read-only verifier and inspection +keys and a bucket/prefix-restricted Restic read/list/write key. Provisioning and +permanent-deletion credentials were absent. The maintenance phase used a 1 MiB +staging mount, a configured 65,536-byte free-space threshold, and disabled cache. +Checkpoint preparation and independent acceptance used a separate central +workspace; that preparation is not covered by the maintenance disk budget. + +A separate changed-plan scenario created a fresh zero-forget plan, then wrote +one eight-byte probe under the same restricted repository prefix. Its single +stale apply was rejected before maintenance-ledger entry, forget, or prune. +All 65 previously observed versions, including the probe, remained exactly +unchanged. Normal pre-validation Restic inspection created and released two +temporary locks, adding two object versions and two hide markers. This is not +a zero-object-mutation guarantee. + +The original fault helper stopped because it required an unchanged version +inventory even for inspection locks. Its blocked result remains preserved. +A read-only continuation bound the exact plan, rejection receipt, before/after +inventories and four new lock versions, without repeating apply. A no-lock +full read-data check passed, both protected snapshot identities remained +visible, all 69 post-inspection inventory rows stayed unchanged, and the +catalog digest was unchanged. The eight-byte probe was deliberately retained. + +This qualifies only the small, zero-forget maintenance exercise. It does not +demonstrate aged reclamation, an elapsed protection window, independently held +off-host escrow, or final remote-server recovery. `protection_qualified` remains +false, the feature ships experimentally in v2.0.4, and #25 remains open. + +#### Live B2 central-maintenance interruption trial + +A separate 2026-10-21 live workstation trial exercised interruption recovery +with one zero-forget plan and one small, approximately 200 KiB fixture; it made +no additional backup sets. A temporary wrapper limited downloads to 1 KiB/s +only for native `prune`, making that phase easier to observe with the small +fixture. One SIGINT was +sent to the owned CLI only after the actual native prune process and its pinned +launch identity, durable prune intent, and a fresh current-lock witness were +verified. The native process stopped and the CLI exited with status 3. The +operation ledger and production prune phase both recorded `uncertain`; the +drill does not claim prune completed. + +After Restic processes were confirmed quiescent, one production read-only +reconciliation recorded `partial_abandoned` and `reconciled_abandoned`, while +the prune outcome remained uncertain. The apply was not replayed. A no-lock full +read and fresh restores of both protected snapshots matched payload, +hidden-file, symlink, and incremental-file expectations. All 69 previously observed provider objects +and markers survived, the catalog stayed unchanged, and no permanent version +deletion was observed. This is evidence for one small interrupted maintenance +operation and recovery check; it does not qualify prune completion, aged +reclamation, the protection window, off-host recovery, or production recovery. + +#### Live Backblaze B2 source trial + +On 2026-10-20, the workstation checkout at commit +`4378e6efbb04de35ad41d377e461fe5d170fab1d` passed a small live B2 S3 source-path +trial in a retained isolated Docker container without the whole checkout or +central state mounted and without central or provisioner credentials. +The 196,682-byte source fixture contained a 192 KiB deterministic payload, a +small hidden synthetic credential file, a symlink, and an incremental file. +The drill used a 1 MiB tmpfs staging mount and a zero-byte Restic cache budget. + +The source key was restricted to one exact bucket and machine prefix. The SDK +profile reported exactly `listBuckets`, `listFiles`, `readFiles`, and +`writeFiles`. Repository initialization, two captures, a repository check, and +restores of both snapshots passed. The older restore omitted the incremental +file; the newer restore contained it. Both restores preserved the hidden file +and symlink. Separate live canaries uploaded, read, and listed an object, then +replaced and hid it while confirming old versions remained available. Seven +forbidden-operation checks were denied and read back: permanent-version delete, +cross-prefix read, cross-prefix write, key administration, bucket +administration, file-retention administration, and legal-hold administration. +The last two used a fresh, explicitly unprotected disposable canary. The central +retention key then deleted that exact 37-byte version, and independent readback +confirmed its absence. This tests authority on an unprotected object; it does +not test the retention planner's aged candidates. Separate protected-object +checks are recorded below. + +This is narrow workstation evidence, not production qualification. No large +backup set was measured. Read-only Restic raw-data statistics reported 197,784 +bytes for the initial snapshot, 197,943 bytes for the incremental snapshot, and +198,997 bytes for their union. The union is smaller than the two per-snapshot +totals combined, demonstrating shared repository data across these snapshots. +These raw-data figures do not measure network transfer bytes, bandwidth, or cost +savings. + +A fresh isolated fixture at commit +`f34945826bdb177dfa0858dd1dbe9470464980ed` subsequently passed initialization, +both captures, a full repository check and both restores. Its final fixture size +was 196,682 bytes. The source container used an archived copy of that exact +commit, the restricted source credential, a 1 MiB staging mount and no cache. +The controller verified the mount allowlist and unchanged source files; central +and provisioning credentials were absent from its mounts and configured +environment. Initialization observed an empty exact-prefix version inventory. + +- Initial capture: interface TX/RX was 221,039/28,033 bytes; provider-reported + stored-version bytes grew by 199,080, including retained noncurrent versions. + Observed additional staging use peaked at 208,896 bytes. +- Incremental capture: interface TX/RX was 22,065/29,811 bytes; stored-version + bytes grew by 2,457. Observed additional staging use peaked at 12,288 bytes. + +Interface counters bracketed each capture, excluding the surrounding inventory +requests, and include protocol and any concurrent interface traffic. Stored-byte +totals sum provider-reported object payload sizes, including noncurrent lock +versions; delete markers are counted separately. Neither measure establishes +provider billing or costs. Local filesystem observations remain unattributed. +The smaller incremental growth is fixture evidence, not a large-dataset or +long-term savings estimate. This run did not inject database-export failures or +qualify reboot recovery, remote-server deployment or aged retention. + +Initial SDK-wrapped retention errors and rejected canary setup attempts remain +preserved separately; they were not counted as passes or replayed. The successful +fresh canary required observed retention `NONE` and legal hold `OFF` before its +authority checks. A verifier lifecycle request initially received HTTP 403; +after adding only `readBucketLifecycleRules` to the separate central identities, +readback confirmed enabled versioning and no applicable lifecycle expiration +rules. The source key stayed unchanged. Backblaze's +[S3-compatible app-key capability guide](https://www.backblaze.com/docs/cloud-storage-s3-compatible-app-keys) +maps that capability to `Get Lifecycle Configuration`; no +`writeBucketLifecycleRules` capability was added. + +A separate central verifier authenticated the source-signed encrypted kit, +reconstructed the exact historical repository versions, checked all data, and +signed an acceptance with a distinct key. The source identity then hid the +configuration object and replaced one data object with invalid bytes. Using the +kit and read-only historical-version access, the central verifier reconstructed +and fully checked the repository again, verified repository/snapshot identity, +and restored the older snapshot with matching payload, hidden file, symlink, +and no later incremental file. Ten repository objects totaling 201,853 bytes +were reconstructed. The named temporary source-password copy was removed only +after baseline acceptance; recovery read its password from the encrypted kit. + +The drill exposed a receipt bug after the kit was written: a checkpoint without +an expiry date could not be inserted into the legacy non-null expiry field. +Checkpoint receipts now permit absent expiry, and an existing owning schema is +migrated transactionally after a private compressed backup. Existing rows are +preserved; schemas with additional constraints are refused. The failed attempt +remains recorded, and its authenticated kit was accepted separately without +resealing or replaying the failed operation. No expiry date is invented, and +absence of an expiry does not qualify a guaranteed protection window. + +Separate fresh disposable versions tested governance retention and legal hold. +The routine central retention identity, which has no governance-bypass +capability, was denied permanent deletion of each exact version. Independent +readback confirmed both survived, and the native metadata reader observed the +future governance expiry and active legal hold. The provisioning identity then +removed protection and deleted only those test canaries. Earlier inconclusive +attempts remain separate and are not counted as successful denial checks. These +checks do not demonstrate aged candidate deletion or a thirty-day recovery window. + +A separate probe created two further 39-byte protected canaries and issued a +versionless S3 delete with the restricted source key. Both governance retention +and legal hold allowed current-name hiding: each acquired exactly one marker, +while independent native inspection verified the original exact version and +unchanged active protection. Current-name reads reported absence. Read-only +reconciliation recorded no uncertain operations, and exact scoped provisioning +cleanup removed only those two uploads and their markers. This confirms that +Object Lock protects historical versions without making current names append-only; +it does not establish a thirty-day window or aged planner behavior. + +Thirty-day elapsed grace, overlapping lifecycle behavior, aged retention, +off-host custody, clean replacement, final remote-server qualification, and all +live AWS behavior remain unproven. Central acceptance still records +`protection_qualified: false`. The v2.0.4 stable support scope remains the +v2.0.3 tested core scope. + +#### Central B2 retention planning rehearsal + +A live rehearsal enrolled a catalog, generated a plan with a fresh +thirty-day first-seen grace period, ran a guarded zero-apply, and reconciled an +empty plan. It counted 10 protected versions in a 31-version inventory; the +before/after inventory matched and no S3 delete was sent. Empty reconciliation +made zero provider reads and does not test uncertain-delete recovery. The S3 +Object Lock configuration read returned ambiguous +`ObjectLockConfigurationNotFoundError`. The new native reader separately passed +live reads of both exact original configuration/data versions, normalizing only +authorized null settings and running even when the S3 lock flag was false. +The production CLI then completed setup and deliberate local schema-3 +enrollment, followed by schema-2 native-metadata planning, zero-candidate apply, +and empty reconciliation against the same existing catalog and repository +lease. It again protected 10 versions among 31, preserved established first-seen +times and the original policy/grant files, and left the version inventory +unchanged. This verifies the production integration with fresh grace; it does +not exercise an aged deletion or an uncertain mutation. No cost/savings, +off-host, or remote-server qualification is established. + +#### Live B2 lifecycle-overlap guard trial + +On 2026-10-21, a new `allPrivate` B2 bucket was created with File Lock enabled, +default retention `NONE`, and two overlapping native lifecycle rules. The +broader prefix hides files after 3,650 days; the nested prefix deletes +noncurrent versions after 3,651 days. Backblaze documents that native rules may +overlap and combine the smallest non-null value for each action +([native lifecycle rules](https://www.backblaze.com/docs/en/cloud-storage-lifecycle-rules)). +Its S3-compatible `Get Lifecycle Configuration` returned the corresponding +three actions: hide after 3,650 days, remove orphan markers, and delete +noncurrent versions after 3,651 days +([S3 Get Lifecycle Configuration](https://www.backblaze.com/apidocs/s3-get-lifecycle-configuration)). +S3 `Put Lifecycle Configuration` rejects overlapping prefixes, so this test +configured the rules natively at bucket creation +([S3 Put Lifecycle Configuration](https://www.backblaze.com/apidocs/s3-put-lifecycle-configuration)). + +A one-hour bucket-scoped reader had exactly `listBuckets`, `readBuckets`, +`readBucketLifecycleRules`, and `readBucketRetentions`; it had no object +read/list/write/delete or administration capabilities. Verification ran as a +nonroot process with a read-only root and fixture mounts, a separate writable +work mount, and no Envman or provisioner credentials. Native readback confirmed +the bucket type, File Lock setting, default retention, and both rules; S3 +readback confirmed enabled versioning and the three mapped actions. The actual +central-retention bucket inspection raised +`CloudError: Enabled bucket lifecycle expiration blocks central retention`. +The read-only inventory observation found no versions in its first entry; that +receipt is bounded to the first page. No object was uploaded, no bucket setting +was updated, and no deletion was attempted. + +This establishes only the configured overlap guard and central-retention +refusal. It does not test elapsed lifecycle expiry, physical reclamation, or +any file-level lock/retention window; no actual file objects were created. +Central acceptance remains `protection_qualified: false`, requests #23–#26 +remain open, and the v2.0.4 stable scope remains the v2.0.3 tested core scope. + +On 2026-10-20 a workstation-only Garage S3 drill captured a 2,184,000-byte file +fixture directly to the emulator using a 1,048,576-byte staging mount, ran an +incremental capture and full repository check, and restored the older snapshot +including a hidden synthetic credential file and a symlink. Initial/incremental +PUT payloads were 4,979/4,113 bytes; internet backup traffic was zero. The source +was intentionally repetitive and small, so these numbers are fixture evidence, +not performance or cost estimates. This does not qualify provider permissions, +independent disaster recovery or an entire Linux machine. + +A second small local drill also refused a SQLite export larger than the 1 MiB +staging capacity before uploading any export data. A subsequent full repository +check passed, preserving the earlier recovery points after the budget failure. +The export source was about 2 MiB; no large backup set or internet transfer was +used. + +#### Aggregate-capacity and active-pressure trial + +On 2026-10-21, code at `d33354a` captured a small, compressible +2,245,463-byte logical baseline to a local Garage S3 emulator. The isolated +source had a read-only root filesystem and read-only fixture inputs. Its total +writable filesystem capacity was 1,769,472 bytes: 1,048,576 for staging, +524,288 for private state, 131,072 for temporary files, and 65,536 for shared +memory. The baseline exceeded that combined capacity, not just staging. +Restic cache was disabled. Capture used a loopback HTTPS proxy; internet backup +traffic was zero. + +A second capture included a new 32-byte trigger. While its native Restic child +was sending a data PUT, a task-owned filler crossed the configured +131,072-byte free-space threshold; the observed minimum was 126,976 bytes. +The runner stopped the child and reported `resource_budget_exceeded`, with +both completion flags false and no replay. A subsequent capture was blocked. +After checked filler removal, read-only reconciliation recorded the failed +capture without repeating upload; its recovery artifacts remained preserved. +The operation ledger also records a successful full read-data check. + +The original harness then stopped at a symlink metadata assertion because +Restic's JSON listing omitted the link target. That blocked result remains +unchanged. Preserved state and staging archives bound a separate read-only +continuation in a new verification container. Its no-lock full read-data check +and verified restore passed. All three files matched source sizes and SHA-256 +digests, including the hidden synthetic credential file and 2,184,000-byte +data file. The restored symlink targeted `data.txt`; the pressure-only trigger +was absent. The continuation did not replay capture or reconciliation. + +This qualifies only the small local scenario. It does not measure normal peak +storage, incompressible-data scale, host memory, persistent staging across +reboot, off-host recovery, or production recovery timing. Reserve checks remain +observed thresholds rather than a hard guarantee. Final remote-server +qualification for #26 remains deferred. + +### Workstation implementation checks - 2026-10-20 + +The initial implementation suite passed **1,057 tests with one skip**, using +eight workers under the shared CPU budget. The subsequent nullable-receipt fix +passed **1,067 tests with one skip**; the native B2 metadata follow-up passed +**1,087 tests with one skip**. Focused schema-2 tests cover authorized-null and +held/retained versions, target and credential binding, generic S3 compatibility, +and changed or unknown native evidence before deletion. Focused regressions and independent source +review covered restricted-key assessment, separately signed central acceptance, +retention dependency loss and uncertain deletion, remote-primary resource +failure, enforced Linux exclusions and mount-namespace drift. Generated CLI +catalogs/schemas, version synchronization, publishing readiness, Markdown and +asset provenance checks passed. Changed Python paths passed Ruff and compilation. + +The subsequent reviewed SQLite coverage and remote-export lifecycle fixes at +`f34945826bdb177dfa0858dd1dbe9470464980ed` passed **1,122 tests with one skip** in +99.31 seconds using eight workers. The skipped legacy Docker restore round trip +suppressed a failed fixture backup; its underlying cause was not identified, so +it is not restore evidence. Focused regressions cover raw sidecar refusal, +transactional restore and metadata reconstruction, retained exports after +interruption or receipt failure, streamed remote-byte verification, crash-window +recovery, explicit digest-bound disposal and legacy reconciliation compatibility. +Generated catalogs/schemas, version, publishing, compilation, Markdown, Ruff and +diff checks passed. + +An offline wheel/source build passed portable source-archive checks and an +isolated installed-wheel smoke test. These are historical checks of a +development checkout still carrying package version 2.0.3; they are not the +v2.0.4 release-candidate validation or published release artifacts, and they do +not claim that the open live qualification work is complete. All tracked README +entry points were reviewed, with current-state links and support boundaries +aligned while preserving historical release records. +The wheel and portable source archive built from the same `f349458` commit also +passed source-content checks and an isolated installed-wheel smoke test. The +private wheel SHA-256 is +`ecef6617633a6547b460109dca7d9264aaa5d0a1607901331842856c9dc83a59`. + +A fresh Ubuntu 24.04.5 KVM guest passed all ten existing user-timer checks across +three boots in 333.9 seconds. The tested development wheel SHA-256 was +`e9ad97c9d9575bc1ca1ca0a0bb1efaf43f6512fc75aa8377d69f00e19df94bbc`. +It exercised enrollment, capture/copy/byte-matched restore, missed-timer +catch-up, disabled timers, overlap refusal, interrupted-operation uncertainty +and no replay. Both repositories remained in that guest. This does not qualify +root covered jobs, live B2 or off-host recovery. The subsequent covered-timer +temporary-source-view correction passed all 29 focused scheduling/coverage +checks; ordinary timer rendering remained unchanged. + +### Existing version-2 behavior + - Strict version-2 JSON configuration rejects duplicate keys, unknown fields, unsupported source types, and inconsistent repository references. Private host bindings and password files are separate from portable policy. @@ -41,8 +777,9 @@ An implemented capability does not establish completion of its entire delivery s ### Native Docker capture and recovery -This implemented path is not production-qualified and is outside the selected -stable v2.0.2 release scope. Docker capture is opt-in and requires a local Docker Unix socket in the private +This implemented path is not production-qualified and is outside the current +stable support scope, which is unchanged from v2.0.3. Docker capture is opt-in +and requires a local Docker Unix socket in the private host bindings. The portable source policy names one container and requires an explicit stop-and-restart window: @@ -290,13 +1027,18 @@ another host using a different ledger. Restic provides its own repository locks, but independent recovery checkpoints additionally need a coordinated quiescent window across all writers. -The v2 retention workflow has no automatic deletion or pruning schedule. Local retention can +The local snapshot-retention workflow has no automatic deletion or pruning schedule. It can forget explicitly reviewed snapshots using restic's daily selection policy, grouped by host and filtered by the job tag. Seven local daily recovery points is the default; this is not a count of the last seven captures. Cloud defaults of thirty daily, eight weekly, and twelve monthly points remain policy, not a qualified provider lifecycle or pruning implementation. +Experimental central cloud retention is a separate catalog/grant and reviewed +plan/apply workflow for noncurrent object versions. It never runs Restic +`forget`/`prune` or compacts current packs. See the +[central retention guide](../cloud-storage.md#central-noncurrent-version-retention-experimental). + ```bash bbackup production --config config.json --bindings bindings.json retention plan --job daily bbackup production --config config.json --bindings bindings.json retention apply \ @@ -543,11 +1285,11 @@ A disposable real-restic capture, independent-password replication, full-data check, and restore preserved exact file bytes, mode 0640, and a symbolic link. These local checks do not qualify a cloud provider or application recovery. -No live B2/AWS provider qualification is recorded. A local Garage S3-compatible -test does not qualify those providers. Provider checks require disposable -authorized resources; without such evidence, B2/AWS remain outside the stable -support scope. Do not replay uncertain backup mutations or change live provider -policy during a recovery. +No B2/AWS production qualification is recorded. The small live B2 source trial +above does not qualify provider protection, and a local Garage S3-compatible +test does not qualify either provider. B2/AWS remain outside the stable support +scope. Do not replay uncertain backup mutations or change live provider policy +during a recovery. The signed [2.0.0-alpha.1 prerelease](https://github.com/CruxExperts/BestBackup/releases/tag/v2.0.0-alpha.1) was published on October 3, 2026 from commit `b10ed84`. GitHub CI passed on Python 3.12, 3.13, and 3.14, and the release workflow built and smoke-tested its wheel and source archive. `SHA256SUMS` and its OpenPGP signature cover the exact downloadable artifacts. The release remains explicitly marked as a prerelease; it does not establish production qualification. @@ -656,9 +1398,11 @@ recorded passing evidence. The following paths are not production-qualified or remain deferred; their implementation does not expand the selected support boundary: -- **B2/AWS:** No live provider capture/check/restore result is recorded. These - implemented paths are not production-qualified and remain outside the selected - stable support boundary. Garage S3 testing is not provider qualification. +- **B2/AWS:** A small live B2 source capture/check/restore trial and denied + forbidden-operation checks are recorded above. Provider protection and + production recovery remain unqualified; AWS has no live result. These paths + remain outside the selected stable support boundary. Garage S3 testing is not + provider qualification. - **Native Docker:** Implemented, but no qualifying capture/restore happy path is recorded. It is not production-qualified and remains outside the selected stable support boundary. diff --git a/docs/production-cli.md b/docs/production-cli.md index f38157f..b8f62cc 100644 --- a/docs/production-cli.md +++ b/docs/production-cli.md @@ -2,7 +2,7 @@ > Generated by `scripts/generate_cli_skills.py` from `bbackup/cli_metadata.py`. -The command contract is stable in v2.0.3. Production qualification remains limited to the tested scope in [version 2 release evidence](development/version-2.md). +The command contract is stable in v2.0.4. Production qualification remains limited to the tested scope in [version 2 release evidence](development/version-2.md). Prefix commands with `bbackup production --config CONFIG --bindings BINDINGS`. @@ -54,6 +54,7 @@ List configured capture jobs. Capture locally and copy explicit successful snapshots. - `--name`: string. +- `--quiet-administrative-window-confirmed`: boolean. ## `repositories list` @@ -102,6 +103,98 @@ Apply only a reviewed local retention plan after explicit approval; never prunes - `--plan-token`: string. - `--approved`: boolean. +## `cloud-retention catalog-enroll` + +Explicitly enroll a new central exact-version retention catalog; never replace a missing established catalog. + +- `--repository`: string. +- `--policy-file`: path. +- `--grant-file`: path. + +## `cloud-retention plan` + +Plan noncurrent object-version deletion from independently accepted checkpoints and private authority. + +- `--repository`: string. +- `--policy-file`: path. +- `--grant-file`: path. +- `--records-file`: path. +- `--quiescent-window-confirmed`: boolean. + +## `cloud-retention apply` + +Apply one reviewed exact-version plan under the shared repository lease; never replay uncertain deletes. + +- `--repository`: string. +- `--policy-file`: path. +- `--grant-file`: path. +- `--records-file`: path. +- `--plan-id`: string. +- `--plan-sha256`: string. +- `--quiescent-window-confirmed`: boolean. + +## `cloud-retention reconcile` + +Reconcile a cloud-version deletion journal using read-only observations without replaying it. + +- `--repository`: string. +- `--policy-file`: path. +- `--plan-file`: path. + +## `cloud-maintenance catalog-enroll` + +Explicitly initialize the central catalog under a current-repository maintenance grant. + +- `--repository`: string. +- `--policy-file`: path. +- `--grant-file`: path. + +## `cloud-maintenance records-enroll` + +Explicitly register independently signed accepted recovery records without changing version-age evidence. + +- `--repository`: string. +- `--policy-file`: path. +- `--grant-file`: path. +- `--records-file`: path. + +## `cloud-maintenance plan` + +Plan exact accepted and unprotected snapshot removal using read-only Restic observations. + +- `--repository`: string. +- `--policy-file`: path. +- `--grant-file`: path. +- `--records-file`: path. +- `--maintenance-bindings-file`: path. +- `--quiescent-window-confirmed`: boolean. + +## `cloud-maintenance apply` + +Apply a reviewed snapshot-forget and no-repack prune plan under one repository lease. + +- `--repository`: string. +- `--policy-file`: path. +- `--grant-file`: path. +- `--records-file`: path. +- `--maintenance-bindings-file`: path. +- `--plan-id`: string. +- `--plan-sha256`: string. +- `--quiescent-window-confirmed`: boolean. + +## `cloud-maintenance reconcile` + +Observe an interrupted maintenance attempt without replaying forget or prune. + +- `--repository`: string. +- `--policy-file`: path. +- `--grant-file`: path. +- `--records-file`: path. +- `--maintenance-bindings-file`: path. +- `--plan-id`: string. +- `--plan-sha256`: string. +- `--quiescent-window-confirmed`: boolean. + ## `restore` Restore and verify files in a new isolated directory. @@ -124,6 +217,44 @@ Render disabled systemd service and persistent timer units. - `--name`: string. - `--executable`: path. +## `service render-system` + +Render root system service and persistent timer units after validating root-controlled executable and configuration paths; never installs them. + +- `--name`: string. +- `--executable`: path. + +## `coverage discover` + +Inventory exact private Linux scope and record a bounded candidate for review. + +- `--name`: string. +- `--quiet-administrative-window-confirmed`: boolean. + +## `coverage review` + +Approve an exact complete Linux coverage candidate for the enrolled machine, user, policy and bindings. + +- `--name`: string. +- `--candidate-digest`: string. +- `--quiet-administrative-window-confirmed`: boolean. + +## `coverage check` + +Check reviewed Linux coverage and structural drift without exposing private paths. + +- `--name`: string. +- `--quiet-administrative-window-confirmed`: boolean. + +## `coverage recovery-evidence` + +Assess private local recovery evidence against the exact current baseline and snapshot; never qualifies whole-machine recovery. + +- `--name`: string. +- `--snapshot`: string. +- `--evidence-file`: optional-string. +- `--quiet-administrative-window-confirmed`: boolean. + ## `service install` Install reviewed user-level units; activate the exact timer only with --activate. @@ -164,6 +295,22 @@ Authenticate and decrypt a recovery kit into a new private directory. - `--target`: path. - `--signer`: string. +## `recovery accept` + +Independently reconstruct and check a historical repository, then sign central acceptance with a separate verifier key. + +- `--kit`: path. +- `--target`: path. +- `--source-signer`: string. +- `--verifier`: string. +- `--repository`: string. +- `--repository-id`: string. +- `--snapshot`: string. +- `--bucket`: string. +- `--endpoint`: string. +- `--prefix`: optional-string. +- `--region`: optional-string. + ## `recovery restore` Restore signed exact historical object versions into a new local repository. @@ -202,6 +349,23 @@ Record an administrator-reviewed outcome for an uncertain operation without repl - `--operation-id`: string. - `--outcome`: string. - `--repository-reviewed`: boolean. +- `--quiescent-window-confirmed`: boolean. + +## `runs retained-exports` + +List bounded private remote capture artifact records, including captures stopped before dispatch; counts and digests are declared, not file-hash verification. + +- `--repository`: string. +- `--limit`: limit. +- `--offset`: offset. + +## `runs discard-exports` + +Explicitly discard the exact retained exports of a reviewed failed or abandoned remote capture, bound to its current private record digest. + +- `--repository`: string. +- `--operation-id`: string. +- `--artifact-record-sha256`: string. ## `databases restore` diff --git a/pyproject.toml b/pyproject.toml index b131216..cca903c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "bbackup" -version = "2.0.3" +version = "2.0.4" description = "Local-first encrypted Linux backups with restic and a visual dashboard" readme = "README.md" requires-python = ">=3.12" diff --git a/scripts/README.md b/scripts/README.md index e7eb2bb..7d6f972 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -38,7 +38,7 @@ It requires restic and never touches user backup repositories. The test container in [Dockerfile.test](../Dockerfile.test) uses the official Python 3.12.15 slim Trixie multiarchitecture index `sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1`. -It installs `uv` 0.11.21 from the wheel and SHA-256 in +It installs `uv` 0.13.4 from the wheel and SHA-256 in [uv-bootstrap.txt](../.github/uv-bootstrap.txt) with pip hash checking. Docker backup, restore, and integration helpers share the official Alpine 3.23.6 multiarchitecture index @@ -51,7 +51,7 @@ arm64 wheel hash is not a native arm64 runtime qualification. `apt-get` and `apk distribution packages at build time, so these recipes are not byte-for-byte reproducible. Sources: [Python official image](https://hub.docker.com/_/python), [Alpine official image](https://hub.docker.com/_/alpine), and -[uv on PyPI](https://pypi.org/project/uv/0.11.21/). +[uv on PyPI](https://pypi.org/project/uv/0.13.4/). ### Rebootable systemd qualification @@ -62,14 +62,18 @@ catch-up after a missed schedule, overlap refusal, and an abrupt guest reset without automatic replay of an uncertain operation. Both repositories share the guest; this does not prove off-host disaster recovery. -The host needs Docker, KVM access, `genisoimage`, `gpgv`, and the Ubuntu cloud-image -public keyring. Review the script's pinned image/toolchain before running it. +The host needs Docker, `genisoimage`, `gpgv`, and the Ubuntu cloud-image public +keyring. KVM hardware virtualization is the default. On hosts without KVM, +`--accelerator tcg` selects slower software emulation and does not expose +`/dev/kvm` to the container. The evidence records the selected accelerator; +an emulated run does not measure native performance. Review the script's pinned +image/toolchain before running it. Supply previously reviewed restic and uv executables, a target-compatible runtime wheelhouse, and hash-locked requirements exported from `uv.lock`: ```bash uv run python scripts/qualify_systemd.py \ - --artifact dist/bbackup-2.0.3-py3-none-any.whl \ + --artifact dist/bbackup-2.0.4-py3-none-any.whl \ --wheel-sha256 '' \ --ubuntu-codename noble \ --wheelhouse /path/to/reviewed-runtime-wheels \ @@ -86,10 +90,22 @@ passwords, guest disks, and machine identifiers remain under ignored See [recorded qualification scope](../docs/development/version-2.md) before making support claims. +Add `--scenario covered-root` to the same command for a separate root-coverage +and replacement rehearsal. It installs units inside disposable guests, reviews +their real Linux inventory, and captures a tiny files/SQLite application fixture. +A fresh replacement overlay receives the small recovery disk and candidate +inputs, then restores data, reconstructs fixture identities and metadata, and +checks application startup. The source operating-system overlay is not attached +to the replacement. Evidence is classified as local emulation; it does not +qualify off-host custody, provider protection or complete-machine recovery. +The default `timer-reboot` scenario remains the ordinary user-timer drill. + ### Build a native Ubuntu package The `build_deb.py` script builds a `.deb` from the supplied bbackup wheel, this repository's `uv.lock`, and a supplied restic 0.19.1 executable. The builder requires an already installed `uv`, `dpkg-deb`, and the `packaging` module in its Python interpreter; on supported Ubuntu/Mint hosts, install `python3-packaging` and invoke `/usr/bin/python3`. It does not download restic, install packages on the host, update `uv.lock`, or install Python packages into system Python. Run it as an unprivileged user. + +The upstream [GO-2026-5932 advisory](https://pkg.go.dev/vuln/GO-2026-5932) concerns restic 0.19.1's separate `self-update` OpenPGP verification path. bbackup packaging does not invoke `restic self-update`: release jobs use versioned archives with fixed SHA-256 digests, and `build_deb.py` checks the supplied executable bytes. This caveat is limited to the unused self-update path; it makes no broader claim about backup or restore behavior. The locked dependency export explicitly uses the same native interpreter that ran the builder, so a repository-level `.python-version` pin cannot force a different Python version. diff --git a/scripts/qualify_systemd.py b/scripts/qualify_systemd.py index 7613758..4572829 100644 --- a/scripts/qualify_systemd.py +++ b/scripts/qualify_systemd.py @@ -485,6 +485,1166 @@ def main(): ''' +# This second guest program deliberately runs through the normal production CLI +# and guest inventory. It is opt-in so the established reboot/timer drill +# above retains its exact default behavior. +COVERED_ROOT_GUEST_STAGE = r"""#!/usr/bin/python3 +import base64, ctypes, ctypes.util, grp, hashlib, json, os, pwd, shutil, signal +import sqlite3, stat, subprocess, sys, time, uuid +from pathlib import Path + +LOG = Path('/var/log/bbackup-covered-root-qualification.log') +INPUTS = Path('/mnt/bbackup-inputs') +RECOVERY = Path('/mnt/bbackup-recovery') +MATERIAL = RECOVERY / 'material' +REPOSITORY = RECOVERY / 'repository' +PASSWORD = MATERIAL / 'repository.password' +CONFIG_DIR = Path('/etc/bbackup/covered-root') +CONFIG = CONFIG_DIR / 'config.v3.json' +BINDINGS = CONFIG_DIR / 'bindings.v3.json' +DECLARATION = MATERIAL / 'coverage-declaration.json' +STATE = Path('/var/lib/bbackup-covered-root-state') +DATA_ROOT = Path('/var/lib/bbackup-covered-root') +FILES = DATA_ROOT / 'files' +DATABASE = DATA_ROOT / 'app.sqlite' +APP_NAME = 'coverageapp' +APP_UNIT = 'bbackup-covered-fixture.service' +JOB_NAME = 'covered' +BIN = Path('/opt/bbackup/venv/bin/bbackup') +RESTIC = Path('/usr/local/libexec/bbackup/restic.real') +EXPECTED_CANDIDATE = '@CANDIDATE_SHA256@' +EXPECTED_RELEASE = '@UBUNTU_RELEASE@' +EXPECTED_PYTHON = '@UBUNTU_PYTHON@' +RESTORE_TARGET = Path('/var/tmp/bbackup-covered-root-restore') +REHEARSAL_ROOT = Path('/var/tmp/bbackup-covered-root-rehearsal') +RESTORE_DIAGNOSTICS = Path('/var/log/bbackup-covered-root-restore-diagnostics') +RESTORE_DIAGNOSTIC_SITE = r'''import os +import sys +import traceback +from bbackup import production_cli + +original_emit = production_cli._emit + +def record_restore_exception(command, data=None, error=None): + if command == 'restore' and isinstance(error, dict) and error.get('code') == 'operation_failed': + details = sys.exc_info() + if details[1] is not None: + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW + try: + descriptor = os.open( + '/var/log/bbackup-covered-root-restore-diagnostics/exception.txt', + flags, 0o600) + with os.fdopen(descriptor, 'w', encoding='utf-8') as stream: + traceback.print_exception(*details, file=stream) + except OSError: + pass + return original_emit(command, data=data, error=error) + +production_cli._emit = record_restore_exception +''' + + +def log(message): + line = str(message) + with LOG.open('a', encoding='utf-8') as stream: + stream.write(line + '\n') + stream.flush() + os.fsync(stream.fileno()) + print(line, flush=True) + + +def run(argv, *, check=True, timeout=120, capture=True, env=None): + log(f'COMMAND start: {argv[0]} {" ".join(argv[1:])}') + result = subprocess.run(argv, text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, timeout=timeout, + env=env, check=False) + output = (result.stdout or '').strip() + log(f'COMMAND rc={result.returncode}: {argv[0]} {" ".join(argv[1:])}') + if output: + log(output[-3000:]) + if check and result.returncode: + raise RuntimeError(f'command failed ({result.returncode}): {argv[0]}') + return result + + +def rootctl(*args, check=True, timeout=90): + return run(['/usr/bin/systemctl', *args], check=check, timeout=timeout) + + +def production(*args, check=True, timeout=180): + env = None + if args and args[0] == 'restore': + RESTORE_DIAGNOSTICS.mkdir(mode=0o700, exist_ok=True) + RESTORE_DIAGNOSTICS.chmod(0o700) + site = RESTORE_DIAGNOSTICS / 'sitecustomize.py' + site.write_text(RESTORE_DIAGNOSTIC_SITE, encoding='utf-8') + site.chmod(0o600) + env = dict(os.environ) + env['PYTHONPATH'] = str(RESTORE_DIAGNOSTICS) + ( + os.pathsep + env['PYTHONPATH'] if env.get('PYTHONPATH') else '') + result = subprocess.run([str(BIN), 'production', '--config', str(CONFIG), + '--bindings', str(BINDINGS), *args], text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=timeout, check=False, env=env) + log(f'PRODUCTION rc={result.returncode}: {args[0]}') + if result.stdout.strip(): + log(result.stdout[-4000:].strip()) + lines = [line for line in result.stdout.splitlines() if line.strip()] + try: + envelope = json.loads(lines[-1]) + except (ValueError, TypeError, IndexError): + raise RuntimeError('production CLI did not return its JSON envelope') from None + if not isinstance(envelope, dict): + raise RuntimeError('production CLI returned a non-object envelope') + if check and (result.returncode != 0 or not envelope.get('success')): + raise RuntimeError(f'production command failed: {args[0]}') + return result.returncode, envelope + + +def private_json(path, value): + path = Path(path) + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + path.write_text(json.dumps(value, sort_keys=True, separators=(',', ':')) + '\n', + encoding='utf-8') + path.chmod(0o600) + return path + + +def read_json(path): + return json.loads(Path(path).read_text(encoding='utf-8')) + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + + +def machine_digest(): + machine = Path('/etc/machine-id').read_text(encoding='ascii').strip().lower() + if len(machine) != 32 or not machine or set(machine) == {'0'}: + raise RuntimeError('guest machine identity is unavailable') + return sha256_bytes(machine.encode('ascii')) + + +def mount_inputs(): + INPUTS.mkdir(mode=0o700, parents=True, exist_ok=True) + if not INPUTS.is_mount(): + run(['/usr/bin/mount', '-o', 'ro', '/dev/disk/by-label/BBAINPUTS', str(INPUTS)]) + candidates = list(INPUTS.glob('bbackup-*.whl')) + if len(candidates) != 1 or sha256_file(candidates[0]) != EXPECTED_CANDIDATE: + raise RuntimeError('candidate input wheel or digest is invalid') + return candidates[0] + + +def mount_recovery(role): + RECOVERY.mkdir(mode=0o700, parents=True, exist_ok=True) + device = Path('/dev/vdb') + if not device.exists(): + raise RuntimeError('dedicated repository/recovery disk is missing') + label = run(['/usr/sbin/blkid', '-s', 'LABEL', '-o', 'value', str(device)], + check=False).stdout.strip() + if role == 'source' and not label: + run(['/usr/sbin/mkfs.ext4', '-F', '-L', 'BBARECOVERY', str(device)], timeout=120) + if role == 'replacement' and label != 'BBARECOVERY': + raise RuntimeError('replacement did not receive the source recovery disk') + if not RECOVERY.is_mount(): + run(['/usr/bin/mount', '-o', 'noatime', '/dev/disk/by-label/BBARECOVERY', str(RECOVERY)]) + (RECOVERY / 'material').mkdir(mode=0o700, exist_ok=True) + (RECOVERY / 'repository').mkdir(mode=0o700, exist_ok=True) + os.chmod(RECOVERY / 'material', 0o700) + os.chmod(RECOVERY / 'repository', 0o700) + + +def install_candidate(candidate): + Path('/usr/local/bin').mkdir(mode=0o755, exist_ok=True) + Path('/usr/local/libexec/bbackup').mkdir(mode=0o755, parents=True, exist_ok=True) + shutil.copyfile(INPUTS / 'uv', '/usr/local/bin/uv') + shutil.copyfile(INPUTS / 'restic', RESTIC) + Path('/usr/local/bin/uv').chmod(0o755) + RESTIC.chmod(0o755) + wrapper_script() + if not Path(EXPECTED_PYTHON).is_file(): + raise RuntimeError('selected Ubuntu Python interpreter is unavailable') + run(['/usr/local/bin/uv', 'venv', '/opt/bbackup/venv', '--python', EXPECTED_PYTHON]) + run(['/usr/local/bin/uv', 'pip', 'install', '--offline', '--no-index', + '--find-links', str(INPUTS / 'wheelhouse'), '--require-hashes', + '--python', '/opt/bbackup/venv/bin/python', '-r', str(INPUTS / 'requirements.txt')], + timeout=300) + run(['/usr/local/bin/uv', 'pip', 'install', '--offline', '--no-index', '--no-deps', + '--python', '/opt/bbackup/venv/bin/python', str(candidate)], timeout=120) + run([str(BIN), '--version']) + for path in (Path('/opt/bbackup'), Path('/opt/bbackup/venv')): + shutil.chown(path, user='root', group='root') + path.chmod(0o755) + user_data = Path('/usr/local/bin') + user_data.chmod(0o755) + + +def wrapper_script(): + wrapper = r'''#!/bin/sh +set -u +for argument do + if [ "$argument" = restore ]; then + umask 077 + diagnostic=/var/log/bbackup-covered-root-restore-diagnostics/restic.stderr + status=/var/log/bbackup-covered-root-restore-diagnostics/restic.status + printf 'started\n' > "$status" + /usr/local/libexec/bbackup/restic.real "$@" 2> "$diagnostic" + code=$? + cat "$diagnostic" >&2 + printf 'exit_code=%s\n' "$code" > "$status" + exit "$code" + fi +done +exec /usr/local/libexec/bbackup/restic.real "$@" +''' + path = Path('/usr/local/bin/restic') + path.write_text(wrapper, encoding='utf-8') + path.chmod(0o755) + + +def app_uid_gid(): + used_uids = {item.pw_uid for item in pwd.getpwall()} + used_gids = {item.gr_gid for item in grp.getgrall()} + for value in range(52000, 53000): + if value not in used_uids and value not in used_gids: + return value, value + raise RuntimeError('no free numeric identity is available for the fixture app') + + +def ensure_app_identity(uid, gid): + try: + group = grp.getgrnam(APP_NAME) + except KeyError: + run(['/usr/sbin/groupadd', '--gid', str(gid), APP_NAME]) + group = grp.getgrnam(APP_NAME) + if group.gr_gid != gid: + raise RuntimeError('fixture application group differs from reviewed numeric identity') + try: + user = pwd.getpwnam(APP_NAME) + except KeyError: + run(['/usr/sbin/useradd', '--uid', str(uid), '--gid', str(gid), + '--home-dir', str(FILES / 'home'), '--shell', '/usr/sbin/nologin', + '--no-create-home', APP_NAME]) + user = pwd.getpwnam(APP_NAME) + if (user.pw_uid, user.pw_gid, user.pw_dir, user.pw_shell) != ( + uid, gid, str(FILES / 'home'), '/usr/sbin/nologin'): + raise RuntimeError('fixture application account differs from its reconstruction recipe') + + +def set_acl_if_available(path): + try: + acl_user = next(user for user in pwd.getpwall() + if user.pw_uid not in (0, os.stat(path).st_uid) + and user.pw_uid >= 1000) + except StopIteration: + return None + command = shutil.which('setfacl') + if command: + result = subprocess.run([command, '-m', f'u:{acl_user.pw_uid}:r--', str(path)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if result.returncode == 0: + return acl_user.pw_uid + library_name = ctypes.util.find_library('acl') + if not library_name: + return None + library = ctypes.CDLL(library_name) + library.acl_from_text.argtypes = [ctypes.c_char_p] + library.acl_from_text.restype = ctypes.c_void_p + library.acl_set_file.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p] + library.acl_set_file.restype = ctypes.c_int + library.acl_free.argtypes = [ctypes.c_void_p] + library.acl_free.restype = ctypes.c_int + text = f'u::rw-,u:{acl_user.pw_uid}:r--,g::r--,m::r--,o::---'.encode('ascii') + acl = library.acl_from_text(text) + if not acl: + return None + try: + return acl_user.pw_uid if library.acl_set_file(os.fsencode(path), 0x8000, acl) == 0 else None + finally: + library.acl_free(acl) + + +def has_fixture_acl(path, uid): + try: + raw = os.getxattr(path, 'system.posix_acl_access') + except OSError: + return False + if len(raw) < 12 or (len(raw) - 4) % 8 or int.from_bytes(raw[:4], 'little') != 2: + return False + return any(int.from_bytes(raw[offset:offset + 2], 'little') == 2 + and int.from_bytes(raw[offset + 2:offset + 4], 'little') == 4 + and int.from_bytes(raw[offset + 4:offset + 8], 'little') == uid + for offset in range(4, len(raw), 8)) + + +def metadata(path): + info = path.lstat() + attrs = [] + for name in sorted(os.listxattr(path, follow_symlinks=False)): + value = os.getxattr(path, name, follow_symlinks=False) + attrs.append({'name': name, 'value_base64': base64.b64encode(value).decode('ascii')}) + return {'uid': info.st_uid, 'gid': info.st_gid, 'mode': info.st_mode & 0o7777, + 'links': info.st_nlink, 'type': ('symlink' if stat.S_ISLNK(info.st_mode) + else 'directory' if stat.S_ISDIR(info.st_mode) else 'file'), + 'target': os.readlink(path) if stat.S_ISLNK(info.st_mode) else None, + 'xattrs': attrs} + + +def create_app_files(uid, gid): + DATA_ROOT.mkdir(mode=0o755, parents=True, exist_ok=True) + if FILES.exists(): + shutil.rmtree(FILES) + FILES.mkdir(mode=0o750) + home = FILES / 'home' + home.mkdir(mode=0o750) + app_code = r'''#!/usr/bin/python3 +import hashlib, json, sqlite3, sys +from pathlib import Path +root = Path(__file__).resolve().parent +config_bytes = (root / 'config.json').read_bytes() +config = json.loads(config_bytes) +secret = (root / config['secret']).read_bytes() +database_path = (root / config['database']).resolve() +with sqlite3.connect(database_path.as_uri() + '?mode=ro', uri=True, timeout=5) as db: + rows = [row[0] for row in db.execute('SELECT value FROM committed ORDER BY rowid')] + integrity = db.execute('PRAGMA quick_check').fetchone()[0] +if integrity != 'ok': + raise SystemExit('fixture SQLite integrity failed') +output = {'schema_version': 1, 'rows': rows, + 'secret_sha256': hashlib.sha256(secret).hexdigest()} +encoded = (json.dumps(output, sort_keys=True, separators=(',', ':')) + '\n').encode() +(root / config['output']).write_bytes(encoded) +''' + (FILES / 'app.py').write_text(app_code, encoding='utf-8') + (FILES / 'app.py').chmod(0o750) + (FILES / 'config.json').write_text(json.dumps({ + 'database': '../app.sqlite', 'secret': 'synthetic.secret', + 'output': 'app-output.json'}, sort_keys=True) + '\n', encoding='utf-8') + (FILES / 'config.json').chmod(0o640) + (FILES / 'synthetic.secret').write_bytes(b'synthetic coverage fixture secret v1\n') + (FILES / 'synthetic.secret').chmod(0o640) + (FILES / 'hardlink-original.txt').write_bytes(b'linked fixture content\n') + (FILES / 'hardlink-original.txt').chmod(0o640) + os.link(FILES / 'hardlink-original.txt', FILES / 'hardlink-copy.txt') + (FILES / 'config-link').symlink_to('config.json') + (FILES / 'xattr-probe.txt').write_bytes(b'xattr fixture\n') + (FILES / 'xattr-probe.txt').chmod(0o640) + (FILES / 'app-output.json').write_bytes(b'') + (FILES / 'app-output.json').chmod(0o600) + for path in FILES.rglob('*'): + if path.is_symlink(): + continue + shutil.chown(path, user=uid, group=gid) + shutil.chown(FILES, user=uid, group=gid) + shutil.chown(home, user=uid, group=gid) + try: + os.setxattr(FILES / 'xattr-probe.txt', 'user.coverage', b'covered-root-fixture-v1') + except (AttributeError, OSError) as exc: + raise RuntimeError('guest filesystem cannot store fixture xattrs') from exc + acl_uid = set_acl_if_available(FILES / 'xattr-probe.txt') + if acl_uid is None: + raise RuntimeError('guest filesystem cannot create the required fixture ACL') + os.chown(FILES / 'xattr-probe.txt', uid, gid) + if not has_fixture_acl(FILES / 'xattr-probe.txt', acl_uid): + raise RuntimeError('required fixture ACL did not survive readback') + acl_applied = True + writer = sqlite3.connect(DATABASE) + writer.execute('PRAGMA journal_mode=WAL') + writer.execute('CREATE TABLE committed(value TEXT NOT NULL)') + writer.execute('INSERT INTO committed VALUES (?)', ('committed fixture row',)) + writer.commit() + writer.execute('INSERT INTO committed VALUES (?)', ('uncommitted fixture row',)) + try: + os.setxattr(DATABASE, 'user.coverage', b'covered-root-sqlite-v1') + except (AttributeError, OSError) as exc: + writer.close() + raise RuntimeError('guest filesystem cannot store SQLite xattrs') from exc + shutil.chown(DATABASE, user=uid, group=gid) + DATABASE.chmod(0o640) + metadata_path = FILES / 'numeric-metadata.json' + metadata_path.write_text('{}\n', encoding='utf-8') + metadata_path.chmod(0o600) + entries = {str(path.relative_to(FILES)): metadata(path) + for path in sorted(FILES.rglob('*'))} + entries['.'] = metadata(FILES) + metadata_doc = {'schema_version': 1, 'entries': entries, + 'acl_applied': acl_applied, + 'sqlite_identity': metadata(DATABASE)} + metadata_path.write_text(json.dumps(metadata_doc, sort_keys=True) + '\n', encoding='utf-8') + fixture_bytes = sum(path.lstat().st_size for path in FILES.rglob('*')) + fixture_bytes += DATABASE.stat().st_size + for suffix in ('-wal', '-shm', '-journal'): + sidecar = Path(str(DATABASE) + suffix) + if sidecar.exists(): + fixture_bytes += sidecar.stat().st_size + if fixture_bytes >= 1024 * 1024: + writer.close() + raise RuntimeError('covered-root logical fixture exceeds 1 MiB') + return writer, acl_applied, fixture_bytes + + +def app_unit_text(): + return ("[Unit]\nDescription=Covered root fixture application\nAfter=local-fs.target\n\n" + "[Service]\nType=oneshot\nRemainAfterExit=yes\n" + "User=coverageapp\nGroup=coverageapp\n" + "ExecStart=/var/lib/bbackup-covered-root/files/app.py\n" + "UMask=0077\nKillMode=control-group\nTimeoutStartSec=45\n" + "NoNewPrivileges=yes\n") + + +def verify_root_enrollment(): + for path in (CONFIG, BINDINGS, DECLARATION): + info = path.stat() + if info.st_uid != 0 or info.st_mode & 0o077: + raise RuntimeError('coverage policy, bindings, and declaration must be root-owned and private') + if STATE.exists() and (STATE.stat().st_uid != 0 or STATE.stat().st_mode & 0o077): + raise RuntimeError('production coverage state is not root-owned and private') + + +def write_unit(path, content): + path = Path(path) + path.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + path.write_text(content, encoding='utf-8') + path.chmod(0o644) + + +def verify_installed_units(*, include_fixture): + unit_dir = Path('/etc/systemd/system') + verify_paths = [unit_dir / f'bbackup-{JOB_NAME}.service', + unit_dir / f'bbackup-{JOB_NAME}.timer'] + if include_fixture: + verify_paths.append(unit_dir / APP_UNIT) + run(['/usr/bin/systemd-analyze', 'verify', *map(str, verify_paths)]) + + +def install_production_units(*, include_fixture=True): + _, rendered = production('service', 'render-system', '--name', JOB_NAME, + '--executable', str(BIN)) + units = rendered.get('data') or {} + expected = {f'bbackup-{JOB_NAME}.service', f'bbackup-{JOB_NAME}.timer'} + if set(units) != expected: + raise RuntimeError('production system unit renderer returned an unexpected unit set') + service = units[f'bbackup-{JOB_NAME}.service'] + timer = units[f'bbackup-{JOB_NAME}.timer'] + if ('User=root' not in service or 'Group=root' not in service + or 'OnCalendar=*-*-* *:*:00' not in timer or 'Persistent=true' not in timer): + raise RuntimeError('production system unit renderer omitted root or timer requirements') + unit_dir = Path('/etc/systemd/system') + write_unit(unit_dir / f'bbackup-{JOB_NAME}.service', service) + write_unit(unit_dir / f'bbackup-{JOB_NAME}.timer', timer) + app_unit = app_unit_text() + write_unit(unit_dir / APP_UNIT, app_unit) + private_json(MATERIAL / 'fixture-app-unit.json', {'text': app_unit}) + rootctl('daemon-reload') + verify_installed_units(include_fixture=include_fixture) + return app_unit + + +def wait_for(predicate, label, timeout=150): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + log(f'PASS {label}') + return + time.sleep(2) + raise RuntimeError(f'timed out waiting for {label}') + + +def unit_success(name, *, retained=False): + result = rootctl('show', name, '--no-pager', '-p', 'Result', '-p', 'ExecMainStatus', + '-p', 'ExecMainStartTimestamp', '-p', 'ExecMainExitTimestamp', + '-p', 'ActiveState', '-p', 'SubState') + props = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line) + active, sub = ('active', 'exited') if retained else ('inactive', 'dead') + return (props.get('Result') == 'success' and props.get('ExecMainStatus') == '0' + and bool(props.get('ExecMainStartTimestamp')) + and bool(props.get('ExecMainExitTimestamp')) + and props.get('ActiveState') == active and props.get('SubState') == sub) + + +def timer_triggered(name): + result = rootctl('show', name, '--no-pager', '-p', 'LastTriggerUSec') + value = result.stdout.strip().split('=', 1)[-1] if '=' in result.stdout else '' + return value not in ('', '0', 'n/a') + + +def fixture_identity(uid, gid): + account = pwd.getpwnam(APP_NAME) + group = grp.getgrnam(APP_NAME) + return {'name': APP_NAME, 'uid': account.pw_uid, 'gid': group.gr_gid, + 'home': account.pw_dir, 'shell': account.pw_shell} + + +def base_inventory(): + users = [{'name': item.pw_name, 'uid': item.pw_uid, 'gid': item.pw_gid, + 'home': item.pw_dir, 'shell': item.pw_shell} + for item in pwd.getpwall() if item.pw_name != APP_NAME] + groups = [{'name': item.gr_name, 'gid': item.gr_gid, + 'members': sorted(item.gr_mem)} + for item in grp.getgrall() if item.gr_name != APP_NAME] + units = [] + root = Path('/etc/systemd/system') + stack = [root] + while stack: + directory = stack.pop() + with os.scandir(directory) as entries: + for entry in sorted(entries, key=lambda item: item.name): + path = Path(entry.path) + info = path.lstat() + if stat.S_ISDIR(info.st_mode): + stack.append(path) + elif path.name.endswith('.service'): + if stat.S_ISLNK(info.st_mode): + kind = 'link:' + sha256_bytes(os.readlink(path).encode( + 'utf-8', 'surrogateescape')) + elif stat.S_ISREG(info.st_mode): + kind = 'regular' + else: + raise RuntimeError('unsupported systemd service inventory type') + units.append({'name': path.name, + 'relative_path': str(path.relative_to(root)), 'kind': kind}) + units.sort(key=lambda item: item['relative_path']) + package_bytes = Path('/var/lib/dpkg/status').read_bytes() + mount_rows = [] + for line in Path('/proc/self/mountinfo').read_text(encoding='utf-8').splitlines(): + before, after = line.split(' - ', 1) + fields, tail = before.split(), after.split() + mount_rows.append({'mount_id': int(fields[0]), 'mountpoint': fields[4], + 'filesystem': tail[0], 'source_sha256': sha256_bytes(tail[1].encode())}) + return {'users': users, 'groups': groups, 'units': units, + 'packages_sha256': sha256_bytes(package_bytes), + 'mounts': mount_rows, 'package_bytes': len(package_bytes)} + + +BASE_AUDIT_SCRIPT = r'''#!/usr/bin/python3 +import grp, hashlib, json, os, pwd, stat, subprocess +from pathlib import Path +material = Path('/mnt/bbackup-recovery/material') +expected = json.loads((material / 'base-inventory.json').read_text()) +users = [{'name': p.pw_name, 'uid': p.pw_uid, 'gid': p.pw_gid, + 'home': p.pw_dir, 'shell': p.pw_shell} + for p in pwd.getpwall() if p.pw_name != 'coverageapp'] +groups = [{'name': g.gr_name, 'gid': g.gr_gid, 'members': sorted(g.gr_mem)} + for g in grp.getgrall() if g.gr_name != 'coverageapp'] +if sorted(users, key=lambda x: x['name']) != sorted(expected['users'], key=lambda x: x['name']): + raise SystemExit('base user reconstruction check failed') +if sorted(groups, key=lambda x: x['name']) != sorted(expected['groups'], key=lambda x: x['name']): + raise SystemExit('base group reconstruction check failed') +package_bytes = Path('/var/lib/dpkg/status').read_bytes() +if hashlib.sha256(package_bytes).hexdigest() != expected['packages_sha256']: + raise SystemExit('base package inventory differs from reviewed image') +root = Path('/etc/systemd/system') +actual = [] +stack = [root] +while stack: + directory = stack.pop() + with os.scandir(directory) as entries: + for entry in entries: + path = Path(entry.path) + info = path.lstat() + if stat.S_ISDIR(info.st_mode): + stack.append(path) + elif path.name.endswith('.service'): + if stat.S_ISLNK(info.st_mode): + kind = 'link:' + hashlib.sha256(os.readlink(path).encode( + 'utf-8', 'surrogateescape')).hexdigest() + elif stat.S_ISREG(info.st_mode): + kind = 'regular' + else: + raise SystemExit('unsupported systemd service inventory type') + actual.append({'name': path.name, 'relative_path': str(path.relative_to(root)), + 'kind': kind}) +actual.sort(key=lambda item: item['relative_path']) +if actual != expected['units']: + raise SystemExit('systemd service reconstruction inventory differs') +for name in sorted({item['name'] for item in actual}): + load = subprocess.run(['/usr/bin/systemctl', 'show', name, '-p', 'LoadState', '--value'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, check=False) + if load.returncode or load.stdout.strip() not in ('loaded', 'masked'): + raise SystemExit('systemd service reconstruction check failed') +''' + + +FIXTURE_REBUILD_SCRIPT = r'''#!/usr/bin/python3 +import base64, hashlib, json, os, pwd, grp, shutil, sqlite3, stat, subprocess, sys +from pathlib import Path +material = Path('/mnt/bbackup-recovery/material') +proof = json.loads((material / 'source-proof.json').read_text()) +if len(sys.argv) != 3 or sys.argv[1] not in ('rehearsal', 'replacement'): + raise SystemExit('fixture reconstruction mode is invalid') +mode, restore_root = sys.argv[1], Path(sys.argv[2]) +prefix = Path('/var/tmp/bbackup-covered-root-rehearsal') if mode == 'rehearsal' else Path('/') +files_source = restore_root / 'var/lib/bbackup-covered-root/files' +manifest_paths = list(restore_root.rglob('bbackup-capture.json')) +if not files_source.is_dir() or len(manifest_paths) != 1: + raise SystemExit('restored fixture files or manifest are missing') +manifest = json.loads(manifest_paths[0].read_text()) +entries = [item for item in manifest['exports'] if item.get('source') == 'database'] +if len(entries) != 1: + raise SystemExit('covered SQLite export is missing from manifest') +entry = entries[0] +if (entry.get('format') != 'sqlite' or entry.get('original_path') != proof['database_path'] + or entry.get('coverage_root_id') != 'database' + or entry.get('baseline_digest') != proof['baseline_digest']): + raise SystemExit('covered SQLite manifest differs from reviewed source') +export_path = restore_root / str(entry['path']).lstrip('/') +if hashlib.sha256(export_path.read_bytes()).hexdigest() != entry['export_sha256']: + raise SystemExit('covered SQLite export hash differs from manifest') +raw_restore = restore_root / str(proof['database_path']).lstrip('/') +if raw_restore.exists() or any(Path(str(raw_restore) + s).exists() + for s in ('-wal', '-shm', '-journal')): + raise SystemExit('raw SQLite database artifact was restored') +files_target = prefix / 'var/lib/bbackup-covered-root/files' +database_target = prefix / 'var/lib/bbackup-covered-root/app.sqlite' +files_target.parent.mkdir(mode=0o755, parents=True, exist_ok=True) +if files_target.exists(): + shutil.rmtree(files_target) +subprocess.run(['/bin/cp', '-a', str(files_source), str(files_target.parent)], check=True) +database_target.parent.mkdir(mode=0o755, parents=True, exist_ok=True) +if database_target.exists(): + database_target.unlink() +shutil.copyfile(export_path, database_target) +identity = entry['reviewed_identity'] +for key in ('uid', 'gid', 'mode', 'links', 'xattrs'): + if identity.get(key) != proof['sqlite_identity'].get(key): + raise SystemExit('reviewed SQLite metadata differs from source evidence') +os.chown(database_target, identity['uid'], identity['gid']) +os.chmod(database_target, identity['mode']) +for attr in identity['xattrs']: + os.setxattr(database_target, attr['name'], base64.b64decode(attr['value_base64'])) +database_info = database_target.lstat() +database_attrs = [{'name': name, 'value_base64': base64.b64encode(os.getxattr( + database_target, name, follow_symlinks=False)).decode('ascii')} + for name in sorted(os.listxattr(database_target, follow_symlinks=False))] +if (database_info.st_uid != identity['uid'] or database_info.st_gid != identity['gid'] + or database_info.st_mode & 0o7777 != identity['mode'] + or database_info.st_nlink != identity['links'] or database_attrs != identity['xattrs']): + raise SystemExit('restored SQLite numeric owner, mode, link or xattr metadata differs') +with sqlite3.connect(database_target.as_uri() + '?mode=ro', uri=True) as database: + if database.execute('PRAGMA quick_check').fetchone() != ('ok',): + raise SystemExit('restored SQLite integrity check failed') + rows = [row[0] for row in database.execute('SELECT value FROM committed ORDER BY rowid')] +if rows != ['committed fixture row']: + raise SystemExit('restored SQLite contains missing or uncommitted rows') +identity_info = proof['app_identity'] +try: + group = grp.getgrnam(identity_info['name']) +except KeyError: + subprocess.run(['/usr/sbin/groupadd', '--gid', str(identity_info['gid']), + identity_info['name']], check=True) + group = grp.getgrnam(identity_info['name']) +if group.gr_gid != identity_info['gid']: + raise SystemExit('fixture application group could not be reconstructed') +try: + user = pwd.getpwnam(identity_info['name']) +except KeyError: + subprocess.run(['/usr/sbin/useradd', '--uid', str(identity_info['uid']), + '--gid', str(identity_info['gid']), '--home-dir', identity_info['home'], + '--shell', identity_info['shell'], '--no-create-home', identity_info['name']], check=True) + user = pwd.getpwnam(identity_info['name']) +if (user.pw_uid, user.pw_gid, user.pw_dir, user.pw_shell) != ( + identity_info['uid'], identity_info['gid'], identity_info['home'], identity_info['shell']): + raise SystemExit('fixture application account differs from reviewed identity') +if mode == 'replacement': + unit = json.loads((material / 'fixture-app-unit.json').read_text())['text'] + unit_path = Path('/etc/systemd/system/bbackup-covered-fixture.service') + unit_path.write_text(unit) + unit_path.chmod(0o644) + subprocess.run(['/usr/bin/systemctl', 'daemon-reload'], check=True) + subprocess.run(['/usr/bin/systemd-analyze', 'verify', str(unit_path)], check=True) + subprocess.run(['/usr/bin/systemctl', 'start', 'bbackup-covered-fixture.service'], check=True) + status = subprocess.run(['/usr/bin/systemctl', 'show', 'bbackup-covered-fixture.service', + '-p', 'Result', '-p', 'ExecMainStatus', '-p', 'ExecMainStartTimestamp', + '-p', 'ExecMainExitTimestamp', '-p', 'ActiveState', '-p', 'SubState'], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, check=True).stdout + props = dict(line.split('=', 1) for line in status.splitlines() if '=' in line) + if (props.get('Result') != 'success' or props.get('ExecMainStatus') != '0' + or not props.get('ExecMainStartTimestamp') or not props.get('ExecMainExitTimestamp') + or props.get('ActiveState') != 'active' or props.get('SubState') != 'exited'): + raise SystemExit('fixture application system service did not complete successfully') +else: + output = files_target / 'app-output.json' + subprocess.run(['/usr/sbin/runuser', '-u', identity_info['name'], '--', + str(files_target / 'app.py')], check=True) +if hashlib.sha256((files_target / 'app-output.json').read_bytes()).hexdigest() != proof['app_output_sha256']: + raise SystemExit('reconstructed fixture application output differs') +metadata = json.loads((files_target / 'numeric-metadata.json').read_text()) +for relative, expected in metadata['entries'].items(): + actual_path = files_target if relative == '.' else files_target / relative + actual = actual_path.lstat() + if (actual.st_uid != expected['uid'] or actual.st_gid != expected['gid'] + or actual.st_mode & 0o7777 != expected['mode'] + or actual.st_nlink != expected['links']): + raise SystemExit('restored numeric owner, mode, or link metadata differs') + if stat.S_ISLNK(actual.st_mode) and os.readlink(actual_path) != expected['target']: + raise SystemExit('restored symlink target differs') + attrs = [{'name': name, 'value_base64': base64.b64encode(os.getxattr( + actual_path, name, follow_symlinks=False)).decode('ascii')} + for name in sorted(os.listxattr(actual_path, follow_symlinks=False))] + if attrs != expected['xattrs']: + raise SystemExit('restored xattr or ACL metadata differs') +hard_a, hard_b = files_target / 'hardlink-original.txt', files_target / 'hardlink-copy.txt' +if hard_a.stat().st_ino != hard_b.stat().st_ino: + raise SystemExit('restored hardlink relationship differs') +if os.readlink(files_target / 'config-link') != 'config.json': + raise SystemExit('restored symlink relationship differs') +''' + + +def inventory_unit_records(): + root = Path('/etc/systemd/system') + result = [] + stack = [root] + visited = 0 + while stack: + directory = stack.pop() + with os.scandir(directory) as entries: + children = sorted((Path(entry.path) for entry in entries), reverse=True) + for path in children: + visited += 1 + if visited > 8192: + raise RuntimeError('systemd service inventory exceeds supported bound') + info = path.lstat() + if stat.S_ISDIR(info.st_mode): + stack.append(path) + elif path.name.endswith('.service'): + if stat.S_ISLNK(info.st_mode): + kind = 'link:' + sha256_bytes(os.readlink(path).encode('utf-8', 'surrogateescape')) + elif stat.S_ISREG(info.st_mode): + kind = 'regular' + else: + raise RuntimeError('unsupported systemd service inventory type') + result.append({'name': path.name, 'relative_path': str(path.relative_to(root)), + 'kind': kind}) + result.sort(key=lambda item: item['relative_path']) + return result + + +def recipe_evidence(recipe_id, procedure, environment='isolated-local-fixture'): + procedure = Path(procedure) + procedure.chmod(0o700) + digest = sha256_file(procedure) + evidence_path = MATERIAL / 'recipes' / f'{recipe_id}.evidence.json' + result = run(['/usr/bin/python3', str(procedure)], timeout=120) + if result.returncode: + raise RuntimeError(f'reconstruction procedure failed: {recipe_id}') + evidence = {'schema_version': 1, 'recipe_id': recipe_id, 'recipe_sha256': digest, + 'environment': environment, + 'checks': {'procedure_executed': True, 'result_verified': True}} + private_json(evidence_path, evidence) + return ({'reason': 'reconstruct from the verified Ubuntu base or covered fixture', + 'consequence': 'reviewed local emulation depends on this exact procedure', + 'recipe_id': recipe_id, 'recipe_sha256': digest}, + {'recipe_id': recipe_id, 'recipe_sha256': digest, + 'procedure_path': str(procedure), 'evidence_path': str(evidence_path), + 'evidence_sha256': sha256_file(evidence_path)}) + + +def make_config_and_bindings(): + scope = {'schema_version': 1, 'approved_roots': ['files', 'database'], + 'require_privileged': True, 'filesystem_boundaries': 'explicit', + 'metadata_expectations': {key: True for key in + ('numeric_owner', 'mode', 'acl', 'xattr', 'links')}} + policy = {'schema_version': 3, + 'repositories': [{'name': 'local', 'role': 'local'}], + 'sources': [{'name': 'files', 'kind': 'files'}, + {'name': 'database', 'kind': 'sqlite'}], + 'jobs': [{'name': JOB_NAME, 'repository': 'local', + 'sources': ['files', 'database'], 'schedule': '*-*-* *:*:00', + 'timeout_seconds': 900, 'coverage_policy': scope}]} + bindings = {'schema_version': 3, 'state_dir': str(STATE), + 'repositories': {'local': {'repository': str(REPOSITORY), + 'password_file': str(PASSWORD)}}, + 'source_bindings': { + 'files': {'paths': [str(FILES)]}, + 'database': {'paths': [str(DATABASE)]}}, + 'coverage_declarations': {JOB_NAME: str(DECLARATION)}} + policy_path = MATERIAL / 'policy-candidate.json' + bindings_path = MATERIAL / 'bindings-candidate.json' + private_json(policy_path, policy) + private_json(bindings_path, bindings) + return scope, policy, bindings, policy_path, bindings_path + + +def file_tree_size(): + total = sum(path.lstat().st_size for path in FILES.rglob('*')) + total += DATABASE.stat().st_size + for suffix in ('-wal', '-shm', '-journal'): + sidecar = Path(str(DATABASE) + suffix) + if sidecar.exists(): + total += sidecar.stat().st_size + return total + + +def source_role(candidate): + mount_inputs() + mount_recovery('source') + os.chown(RECOVERY, 0, 0) + install_candidate(candidate) + uid, gid = app_uid_gid() + run(['/usr/sbin/groupadd', '--gid', str(gid), APP_NAME]) + run(['/usr/sbin/useradd', '--uid', str(uid), '--gid', str(gid), + '--home-dir', str(FILES / 'home'), '--shell', '/usr/sbin/nologin', + '--no-create-home', APP_NAME]) + writer, acl_applied, logical_size = create_app_files(uid, gid) + password = (uuid.uuid4().hex + uuid.uuid4().hex).encode('ascii') + b'\n' + PASSWORD.write_bytes(password) + PASSWORD.chmod(0o600) + _, _, _, policy_path, bindings_path = make_config_and_bindings() + _, setup = production('setup', '--configuration-file', str(policy_path), + '--bindings-file', str(bindings_path)) + if setup.get('data', {}).get('configured') is not True: + writer.close() + raise RuntimeError('root-owned schema-three setup did not complete') + install_production_units() + rootctl('start', APP_UNIT) + wait_for(lambda: unit_success(APP_UNIT, retained=True), + 'fixture application systemd invocation') + app_output = FILES / 'app-output.json' + app_identity = fixture_identity(uid, gid) + app_output_hash = sha256_file(app_output) + logical_size = file_tree_size() + if logical_size >= 1024 * 1024: + writer.close() + raise RuntimeError('covered-root logical fixture exceeds 1 MiB') + source_machine = machine_digest() + fixture_meta = read_json(FILES / 'numeric-metadata.json') + sqlite_info = metadata(DATABASE) + source_proof = {'schema_version': 1, 'source_machine_digest': source_machine, + 'app_identity': app_identity, 'database_path': str(DATABASE), + 'sqlite_identity': sqlite_info, 'app_output_sha256': app_output_hash, + 'secret_sha256': sha256_file(FILES / 'synthetic.secret'), + 'fixture_logical_bytes': logical_size, 'acl_applied': acl_applied, + 'fixture_metadata': fixture_meta, 'baseline_digest': '0' * 64, + 'snapshot_id': None, 'capture_manifest_sha256': None, + 'sqlite_export_sha256': None} + private_json(MATERIAL / 'source-proof.json', source_proof) + # The actual guest inventory is read after all application and production + # units are rendered and installed, exactly as production discovery reads it. + inventory = base_inventory() + private_json(MATERIAL / 'base-inventory.json', inventory) + (MATERIAL / 'recipes').mkdir(mode=0o700, parents=True, exist_ok=True) + base_proc = MATERIAL / 'recipes' / 'base-reconstruction.py' + base_proc.write_text(BASE_AUDIT_SCRIPT, encoding='utf-8') + base_proc.chmod(0o700) + # The base recipe is exercised now, and again on the clean replacement. + base_recipe, base_evidence = recipe_evidence('base_reconstruction', base_proc) + fixture_proc = MATERIAL / 'recipes' / 'fixture-reconstruction.py' + fixture_proc.write_text(FIXTURE_REBUILD_SCRIPT, encoding='utf-8') + fixture_proc.chmod(0o700) + fixture_evidence = _exercise_fixture_recipe(fixture_proc, source_proof) + # Build the reviewed declaration after observing real passwd, group, mount, + # package and systemd inventories; these paths stay private to the guest. + users = inventory['users'] + identity_decisions = [{'name': item['name'], 'home': item['home'], + 'recipe': base_recipe} for item in users + if Path(item['home']).is_absolute() and not ( + Path(item['home']) == FILES or FILES in Path(item['home']).parents)] + mounts = [] + for item in inventory['mounts']: + mountpoint = Path(item['mountpoint']) + if mountpoint == FILES or FILES in mountpoint.parents or mountpoint == DATABASE: + mounts.append({'mount_id': item['mount_id'], 'mountpoint': str(mountpoint), + 'decision': 'include'}) + applications = [] + seen_units = set() + for index, item in enumerate(inventory_unit_records()): + stem = item['name'][:-len('.service')] + if stem in seen_units: + continue + seen_units.add(stem) + if stem == APP_UNIT[:-len('.service')]: + continue + apps = [{'id': f'guest_unit_{index:03d}', 'unit': stem, + 'source_id': 'files', 'state_roots': [], + 'startup_recipe': base_recipe}] + applications.extend(apps) + fixture_recipe_id = 'fixture_reconstruction' + fixture_digest = sha256_file(fixture_proc) + fixture_recipe = {'reason': 'recreate the tiny covered app and its SQLite state', + 'consequence': 'replacement must reapply numeric metadata and verify startup output', + 'recipe_id': fixture_recipe_id, 'recipe_sha256': fixture_digest} + applications.extend([ + {'id': 'fixture_files', 'unit': APP_UNIT[:-len('.service')], + 'source_id': 'files', 'state_roots': [str(FILES)], + 'startup_recipe': fixture_recipe}, + {'id': 'fixture_database', 'unit': APP_UNIT[:-len('.service')], + 'source_id': 'database', 'state_roots': [str(DATABASE)], + 'startup_recipe': fixture_recipe}, + ]) + declaration = {'schema_version': 1, + 'roots': [{'id': 'files', 'path': str(FILES), 'source_id': 'files', 'adapter': 'files'}, + {'id': 'database', 'path': str(DATABASE), + 'source_id': 'database', 'adapter': 'sqlite'}], + 'exclusions': [], 'mount_decisions': mounts, + 'identity_decisions': identity_decisions, 'applications': applications, + 'package_reconstruction': base_recipe, + 'reconstruction_evidence': [base_evidence, fixture_evidence]} + private_json(DECLARATION, declaration) + production('service', 'enroll') + verify_root_enrollment() + production('repositories', 'init', '--name', 'local', timeout=300) + discovered = production('coverage', 'discover', '--name', JOB_NAME)[1].get('data', {}) + if discovered.get('coverage_status') != 'complete': + raise RuntimeError('actual production coverage discovery has unresolved inventory gaps') + reviewed = production('coverage', 'review', '--name', JOB_NAME, + '--candidate-digest', discovered['candidate_digest'])[1].get('data', {}) + if reviewed.get('coverage_status') != 'complete': + raise RuntimeError('actual production coverage review did not complete') + check = production('coverage', 'check', '--name', JOB_NAME)[1].get('data', {}) + if check.get('coverage_status') != 'complete' or not check.get('capture_ready'): + raise RuntimeError('actual production coverage check is not capture-ready') + source_proof['baseline_digest'] = reviewed['baseline_digest'] + source_proof['candidate_digest'] = reviewed['candidate_digest'] + private_json(MATERIAL / 'source-proof.json', source_proof) + # Evidence binds the exact final recipe digest and files, not a descriptive + # label. The recipe was run before both reviews above. + rootctl('enable', '--now', f'bbackup-{JOB_NAME}.timer') + timer = f'bbackup-{JOB_NAME}.timer' + service = f'bbackup-{JOB_NAME}.service' + wait_for(lambda: timer_triggered(timer) and unit_success(service), + 'production root systemd timer completed a covered capture', timeout=170) + check = production('coverage', 'check', '--name', JOB_NAME)[1].get('data', {}) + if check.get('coverage_status') != 'complete': + raise RuntimeError('coverage drifted during the production root timer capture') + with sqlite3.connect(STATE / 'snapshots.sqlite3') as receipts: + rows = receipts.execute('SELECT snapshot,operation FROM snapshots ' + 'WHERE repository=? AND job=? ORDER BY rowid DESC', ('local', JOB_NAME)).fetchall() + snapshots = run([str(RESTIC), '-r', str(REPOSITORY), '--password-file', str(PASSWORD), + '--no-cache', '--json', 'snapshots'], timeout=120) + listed = json.loads(snapshots.stdout) + matching = [item for item in listed if f'bbackup-job={JOB_NAME}' in (item.get('tags') or [])] + if not rows or not matching: + raise RuntimeError('timer capture lacks an encrypted repository snapshot or receipt') + snapshot_id = matching[-1]['id'] + receipt = next((row for row in rows if row[0] == snapshot_id), None) + if receipt is None: + raise RuntimeError('timer snapshot has no matching production receipt') + restore = production('restore', '--repository', 'local', '--snapshot', snapshot_id, + '--target', str(RESTORE_TARGET), timeout=300)[1].get('data', {}) + if restore.get('files_verified') is not True: + raise RuntimeError('production restore did not verify the timer snapshot') + manifest_paths = list(RESTORE_TARGET.rglob('bbackup-capture.json')) + if len(manifest_paths) != 1: + raise RuntimeError('production restore did not yield one encrypted capture manifest') + manifest_path = manifest_paths[0] + manifest = read_json(manifest_path) + exports = [item for item in manifest['exports'] if item.get('source') == 'database'] + if len(exports) != 1: + raise RuntimeError('capture manifest has no unique covered SQLite export') + export = exports[0] + if (export.get('format') != 'sqlite' or export.get('original_path') != str(DATABASE) + or export.get('coverage_root_id') != 'database' + or export.get('baseline_digest') != reviewed['baseline_digest']): + raise RuntimeError('SQLite export does not match the reviewed source and baseline') + export_file = RESTORE_TARGET / str(export['path']).lstrip('/') + export_digest = sha256_file(export_file) + if export_digest != export.get('export_sha256'): + raise RuntimeError('SQLite export SHA-256 differs from encrypted capture manifest') + with sqlite3.connect(export_file) as database: + if database.execute('PRAGMA quick_check').fetchone() != ('ok',): + raise RuntimeError('exported SQLite database failed integrity check') + rows_data = [row[0] for row in database.execute( + 'SELECT value FROM committed ORDER BY rowid')] + if rows_data != ['committed fixture row']: + raise RuntimeError('transactional SQLite export included an uncommitted row') + proof_restore = RESTORE_TARGET / str(DATABASE).lstrip('/') + sidecars = [Path(str(proof_restore) + suffix) for suffix in ('-wal', '-shm', '-journal')] + if proof_restore.exists() or any(path.exists() for path in sidecars): + raise RuntimeError('production restore included a raw SQLite file or sidecar') + manifest_digest = sha256_file(manifest_path) + # Restic dump uses the manifest's archived path. Resolve it from metadata + # and compare a direct encrypted-repository read with production restore. + snapshots_meta = run([str(RESTIC), '-r', str(REPOSITORY), '--password-file', str(PASSWORD), + '--no-cache', 'cat', 'snapshot', snapshot_id], timeout=120) + snapshot_meta = json.loads(snapshots_meta.stdout) + manifest_archive = [item for item in snapshot_meta.get('paths', []) + if Path(item).name == 'bbackup-capture.json'] + if len(manifest_archive) != 1: + raise RuntimeError('encrypted repository snapshot has no unique capture manifest') + direct_manifest = run([str(RESTIC), '-r', str(REPOSITORY), '--password-file', str(PASSWORD), + '--no-cache', 'dump', snapshot_id, manifest_archive[0]], timeout=120) + if sha256_bytes(direct_manifest.stdout.encode()) != manifest_digest: + raise RuntimeError('production restore manifest differs from encrypted repository content') + source_proof.update({'snapshot_id': snapshot_id, + 'capture_manifest_sha256': manifest_digest, 'sqlite_export_sha256': export_digest, + 'receipt_operation': receipt[1], 'source_coverage_status': check['coverage_status']}) + private_json(MATERIAL / 'source-proof.json', source_proof) + logical_size = file_tree_size() + if logical_size != source_proof['fixture_logical_bytes']: + raise RuntimeError('covered-root logical fixture changed during capture') + writer.rollback() + writer.close() + rootctl('disable', '--now', timer) + proof_public = {'source_machine_digest': source_machine, 'baseline_digest': reviewed['baseline_digest'], + 'candidate_digest': reviewed['candidate_digest'], 'snapshot_id': snapshot_id, + 'candidate_wheel_sha256': EXPECTED_CANDIDATE, + 'capture_manifest_sha256': manifest_digest, 'sqlite_export_sha256': export_digest, + 'fixture_logical_bytes': logical_size, 'acl_applied': acl_applied} + proof_public['root_timer_invocation'] = { + 'unit': service, 'result': 'success', 'exit_status': 0, + 'start_observed': True, 'exit_observed': True} + log('SOURCE_COMPLETE ' + json.dumps(proof_public, sort_keys=True)) + os.sync() + rootctl('poweroff', timeout=10) + + +def _exercise_fixture_recipe(procedure, proof): + # Build the same path shape that a Restic restore returns, using SQLite's + # transactional backup API and the real fixture files. The reconstruction + # procedure must execute and validate metadata, committed rows and app output. + if REHEARSAL_ROOT.exists(): + shutil.rmtree(REHEARSAL_ROOT) + if REHEARSAL_ROOT.parent.exists(): + shutil.rmtree(REHEARSAL_ROOT.parent / 'bbackup-covered-root-rehearsal-restore', + ignore_errors=True) + restore_tree = REHEARSAL_ROOT.parent / 'bbackup-covered-root-rehearsal-restore' + restore_tree.mkdir(mode=0o700) + target_files = restore_tree / str(FILES).lstrip('/') + target_files.parent.mkdir(mode=0o755, parents=True) + run(['/bin/cp', '-a', str(FILES), str(target_files.parent)], timeout=60) + archive_path = Path('/var/lib/bbackup-covered-root-state/rehearsal/database.sqlite') + archive_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + archive_path.unlink(missing_ok=True) + with sqlite3.connect(DATABASE.as_uri() + '?mode=ro', uri=True) as source: + with sqlite3.connect(archive_path) as target: + source.backup(target) + archive_digest = sha256_file(archive_path) + manifest = {'schema_version': 2, 'exports': [{ + 'source': 'database', 'kind': 'sqlite', 'format': 'sqlite', + 'path': str(archive_path), 'coverage_root_id': 'database', + 'original_path': str(DATABASE), 'reviewed_identity': proof['sqlite_identity'], + 'recipe_digest': '0' * 64, 'baseline_digest': proof['baseline_digest'] or '0' * 64, + 'export_sha256': archive_digest}]} + manifest_path = archive_path.parent / 'bbackup-capture.json' + private_json(manifest_path, manifest) + restore_manifest = restore_tree / str(manifest_path).lstrip('/') + restore_manifest.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(manifest_path, restore_manifest) + restore_archive = restore_tree / str(archive_path).lstrip('/') + restore_archive.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(archive_path, restore_archive) + result = run(['/usr/bin/python3', str(procedure), 'rehearsal', str(restore_tree)], + timeout=120) + if result.returncode: + raise RuntimeError('fixture reconstruction recipe rehearsal failed') + output = REHEARSAL_ROOT / 'var/lib/bbackup-covered-root/files/app-output.json' + if not output.is_file() or sha256_file(output) != proof['app_output_sha256']: + raise RuntimeError('fixture reconstruction recipe output was not verified') + evidence_path = MATERIAL / 'recipes' / 'fixture_reconstruction.evidence.json' + evidence_doc = {'schema_version': 1, 'recipe_id': 'fixture_reconstruction', + 'recipe_sha256': sha256_file(procedure), 'environment': 'isolated-local-fixture', + 'checks': {'procedure_executed': True, 'result_verified': True}} + private_json(evidence_path, evidence_doc) + return {'recipe_id': 'fixture_reconstruction', + 'recipe_sha256': sha256_file(procedure), + 'procedure_path': str(procedure), + 'evidence_path': str(evidence_path), + 'evidence_sha256': sha256_file(evidence_path)} + + +def replacement_role(candidate): + mount_inputs() + mount_recovery('replacement') + install_candidate(candidate) + proof = read_json(MATERIAL / 'source-proof.json') + replacement_machine = machine_digest() + if replacement_machine == proof['source_machine_digest']: + raise RuntimeError('replacement machine identity is not distinct from source') + CONFIG_DIR.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(MATERIAL / 'policy-candidate.json', CONFIG) + shutil.copyfile(MATERIAL / 'bindings-candidate.json', BINDINGS) + CONFIG.chmod(0o600) + BINDINGS.chmod(0o600) + # Deliberate replacement enrollment creates machine-local state in the new + # overlay. No source OS overlay or source enrollment files are attached. + production('service', 'enroll') + verify_root_enrollment() + install_production_units(include_fixture=False) + base_proc = MATERIAL / 'recipes' / 'base-reconstruction.py' + run(['/usr/bin/python3', str(base_proc)], timeout=120) + if RESTORE_TARGET.exists(): + shutil.rmtree(RESTORE_TARGET) + restored = production('restore', '--repository', 'local', '--snapshot', + proof['snapshot_id'], '--target', str(RESTORE_TARGET), timeout=300)[1].get('data', {}) + if restored.get('files_verified') is not True: + raise RuntimeError('production replacement restore did not verify files') + fixture_proc = MATERIAL / 'recipes' / 'fixture-reconstruction.py' + # Refresh the baseline value in the persistent source proof before the + # exact reviewed recipe validates the encrypted capture manifest. + result = run(['/usr/bin/python3', str(fixture_proc), 'replacement', str(RESTORE_TARGET)], + timeout=180) + if result.returncode: + raise RuntimeError('fixture reconstruction and startup recipe failed on replacement') + verify_installed_units(include_fixture=True) + manifest_paths = list(RESTORE_TARGET.rglob('bbackup-capture.json')) + manifest_digest = sha256_file(manifest_paths[0]) + proof_public = {'replacement_machine_digest': replacement_machine, + 'source_machine_digest': proof['source_machine_digest'], + 'candidate_wheel_sha256': EXPECTED_CANDIDATE, + 'baseline_digest': proof['baseline_digest'], 'candidate_digest': proof['candidate_digest'], + 'snapshot_id': proof['snapshot_id'], 'capture_manifest_sha256': manifest_digest, + 'sqlite_export_sha256': proof['sqlite_export_sha256'], + 'fixture_logical_bytes': proof['fixture_logical_bytes'], 'acl_applied': proof['acl_applied'], + 'checks': {'distinct_machine_identities': True, + 'root_owned_schema3_enrollment': True, + 'actual_guest_inventory_and_base_reconstruction': True, + 'production_restore': True, + 'encrypted_capture_manifest_verified': True, + 'sqlite_transactional_export_hash_and_integrity': True, + 'committed_rows_only': True, 'raw_database_and_sidecars_excluded': True, + 'numeric_owner_mode_link_xattr_acl_restore': True, + 'fixture_account_reconstructed': True, + 'synthetic_secret_and_application_output': True, + 'fixture_logical_size_under_1_mib': proof['fixture_logical_bytes'] < 1024 * 1024}} + if not all(proof_public['checks'].values()): + raise RuntimeError('replacement proof contains a failed check') + output = MATERIAL / 'replacement-proof.json' + private_json(output, proof_public) + log('COVERED_ROOT_RESULT ' + json.dumps(proof_public, sort_keys=True)) + os.sync() + rootctl('poweroff', timeout=10) + + +def main(): + if os.geteuid() != 0: + raise RuntimeError('covered-root qualification guest must run as root') + if len(sys.argv) != 2 or sys.argv[1] not in ('source', 'replacement'): + raise RuntimeError('covered-root guest role is invalid') + role = sys.argv[1] + candidate = mount_inputs() + # mount_recovery is called by each role after the candidate input identity + # has been checked, keeping the replacement's only data handoff explicit. + log(f'BOOT_ROLE {role} machine_digest={machine_digest()}') + if role == 'source': + source_role(candidate) + else: + replacement_role(candidate) + + +if __name__ == '__main__': + try: + main() + except Exception as exc: + log(f'FAIL {type(exc).__name__}: {exc}') + try: + os.sync() + rootctl('poweroff', check=False, timeout=10) + except Exception: + pass + raise +""" + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: @@ -652,6 +1812,56 @@ def _create_seed(stage: Path, candidate_digest: str, codename: str) -> Path: return _write_iso(stage, seed_dir, "seed.iso", "cidata") +def _create_covered_root_seed(stage: Path, candidate_digest: str, codename: str, + role: str) -> Path: + if role not in {"source", "replacement"}: + raise ValueError("covered-root seed role must be source or replacement") + seed_dir = stage / f"covered-root-seed-{role}" + seed_dir.mkdir(mode=0o700) + guest = (COVERED_ROOT_GUEST_STAGE.replace("@CANDIDATE_SHA256@", candidate_digest) + .replace("@UBUNTU_RELEASE@", codename) + .replace("@UBUNTU_PYTHON@", UBUNTU_PYTHON[codename])) + service = ( + "[Unit]\nDescription=Task-owned covered-root qualification\n" + "After=network-online.target\nWants=network-online.target\n\n" + "[Service]\nType=oneshot\n" + f"ExecStart=/usr/bin/python3 /usr/local/sbin/bbackup-covered-root-qualification.py {role}\n" + "StandardOutput=journal+console\nStandardError=inherit\n" + "TimeoutStartSec=infinity\n\n[Install]\nWantedBy=multi-user.target\n" + ) + failure_command = ( + "systemctl start bbackup-covered-root-qualification.service || { " + "systemctl status --no-pager --full bbackup-covered-root-qualification.service; " + "journalctl --no-pager -n 40 -u bbackup-covered-root-qualification.service; " + "systemctl poweroff; exit 1; }" + ) + user = ( + "#cloud-config\n" + "write_files:\n" + " - path: /usr/local/sbin/bbackup-covered-root-qualification.py\n" + " owner: root:root\n" + " permissions: '0700'\n" + " encoding: b64\n" + f" content: {base64.b64encode(guest.encode()).decode()}\n" + " - path: /etc/systemd/system/bbackup-covered-root-qualification.service\n" + " owner: root:root\n" + " permissions: '0644'\n" + " content: |\n" + + "".join(" " + line + "\n" for line in service.splitlines()) + + "runcmd:\n" + " - [systemctl, daemon-reload]\n" + " - [systemctl, enable, bbackup-covered-root-qualification.service]\n" + f" - [sh, -c, {json.dumps(failure_command)}]\n" + ) + (seed_dir / "user-data").write_text(user, encoding="utf-8") + (seed_dir / "meta-data").write_text( + f"instance-id: bbackup-covered-root-{role}-{uuid.uuid4().hex}\n" + f"local-hostname: bbackup-covered-root-{role}\n", encoding="ascii") + for child in seed_dir.iterdir(): + child.chmod(0o600) + return _write_iso(stage, seed_dir, f"covered-root-seed-{role}.iso", "cidata") + + def _container_image() -> None: inspect = ["docker", "image", "inspect", "--format", "{{.Os}}/{{.Architecture}} {{json .RepoDigests}}", ALPINE_IMAGE] @@ -674,14 +1884,16 @@ def _container_image() -> None: def _docker_qualification(stage: Path, image: Path, seed: Path, inputs_iso: Path, - timeout: int) -> str: + timeout: int, *, accelerator: str = "kvm") -> str: + if accelerator not in {"kvm", "tcg"}: + raise ValueError("unsupported guest accelerator") name = f"bbackup-systemd-qualification-{uuid.uuid4().hex[:12]}" existing = _run(["docker", "container", "inspect", name]) if existing.returncode == 0: raise RuntimeError("task container name collision; refuse to touch it") docker_command = [ "docker", "run", "--rm", "--name", name, "--platform=linux/amd64", - "--device=/dev/kvm:/dev/kvm:rwm", + *(["--device=/dev/kvm:/dev/kvm:rwm"] if accelerator == "kvm" else []), "--mount", f"type=bind,src={stage},dst=/work", "--mount", f"type=bind,src={image},dst=/ubuntu.img,readonly", ALPINE_IMAGE, @@ -693,7 +1905,8 @@ def _docker_qualification(stage: Path, image: Path, seed: Path, inputs_iso: Path "for p in qemu-10.1.5-r0 qemu-img-10.1.5-r0 qemu-system-x86_64-10.1.5-r0 seabios-1.17.0-r0; do " "grep -F \"$p \" /work/apk-installed.txt >/dev/null || { echo \"missing pinned $p\"; exit 21; }; done; " "qemu-img create -q -f qcow2 -F qcow2 -b /ubuntu.img /work/guest.qcow2 16G; " - "exec qemu-system-x86_64 -machine q35,accel=kvm -cpu host -smp 2 -m 4096 " + f"exec qemu-system-x86_64 -machine q35,accel={accelerator} " + f"-cpu {'host' if accelerator == 'kvm' else 'max'} -smp 2 -m 4096 " "-drive file=/work/guest.qcow2,if=virtio,format=qcow2 " "-drive file=/work/seed.iso,media=cdrom,readonly=on " "-drive file=/work/inputs.iso,media=cdrom,readonly=on " @@ -718,6 +1931,86 @@ def _docker_qualification(stage: Path, image: Path, seed: Path, inputs_iso: Path raise +def _docker_covered_root_qualification(stage: Path, image: Path, source_seed: Path, + replacement_seed: Path, inputs_iso: Path, + timeout: int, *, accelerator: str = "kvm") -> str: + if accelerator not in {"kvm", "tcg"}: + raise ValueError("unsupported guest accelerator") + if source_seed.name == replacement_seed.name: + raise ValueError("source and replacement require distinct cloud-init seed images") + name = f"bbackup-covered-root-qualification-{uuid.uuid4().hex[:12]}" + existing = _run(["docker", "container", "inspect", name]) + if existing.returncode == 0: + raise RuntimeError("task container name collision; refuse to touch it") + qemu_flags = ( + f"-machine q35,accel={accelerator} " + f"-cpu {'host' if accelerator == 'kvm' else 'max'} -smp 2 -m 4096 " + ) + common = ( + "-drive file=/work/{overlay},if=virtio,format=qcow2 " + "-drive file=/work/recovery.raw,if=virtio,format=raw " + "-drive file=/work/{seed},media=cdrom,readonly=on " + "-drive file=/work/{inputs},media=cdrom,readonly=on " + "-netdev user,id=net0 -device virtio-net-pci,netdev=net0 " + "-display none -serial stdio -monitor none -no-reboot" + ) + source_qemu = ("qemu-system-x86_64 " + qemu_flags + + common.format(overlay="source-guest.qcow2", seed=source_seed.name, + inputs=inputs_iso.name) + + " > /work/source-serial.log 2>&1") + replacement_qemu = ("qemu-system-x86_64 " + qemu_flags + + common.format(overlay="replacement-guest.qcow2", seed=replacement_seed.name, + inputs=inputs_iso.name) + + " > /work/replacement-serial.log 2>&1") + shell = ( + "apk update >/dev/null; " + "apk add --no-cache qemu=10.1.5-r0 qemu-img=10.1.5-r0 " + "qemu-system-x86_64=10.1.5-r0 seabios=1.17.0-r0 >/dev/null; " + "apk list -I > /work/apk-installed.txt; " + "for p in qemu-10.1.5-r0 qemu-img-10.1.5-r0 qemu-system-x86_64-10.1.5-r0 seabios-1.17.0-r0; do " + "grep -F \"$p \" /work/apk-installed.txt >/dev/null || { echo \"missing pinned $p\"; exit 21; }; done; " + "qemu-img create -q -f raw /work/recovery.raw 256M; " + "qemu-img create -q -f qcow2 -F qcow2 -b /ubuntu.img /work/source-guest.qcow2 16G; " + "qemu-img create -q -f qcow2 -F qcow2 -b /ubuntu.img /work/replacement-guest.qcow2 16G; " + f"{source_qemu}; " + "grep -Fq 'SOURCE_COMPLETE {' /work/source-serial.log || " + "{ echo 'source guest ended without completion proof'; exit 22; }; " + f"{replacement_qemu}" + ) + docker_command = [ + "docker", "run", "--rm", "--name", name, "--platform=linux/amd64", + *(["--device=/dev/kvm:/dev/kvm:rwm"] if accelerator == "kvm" else []), + "--mount", f"type=bind,src={stage},dst=/work", + "--mount", f"type=bind,src={image},dst=/ubuntu.img,readonly", + ALPINE_IMAGE, "sh", "-ec", shell, + ] + try: + result = _run(docker_command, timeout=timeout) + container_output = result.stdout or "" + apk_log = stage / "apk-installed.txt" + if apk_log.exists() and apk_log.stat().st_uid == os.geteuid(): + apk_log.chmod(0o600) + logs = [] + for role in ("source", "replacement"): + serial = stage / f"{role}-serial.log" + if not serial.is_file(): + serial.write_text("", encoding="utf-8") + if serial.stat().st_uid == os.geteuid(): + serial.chmod(0o600) + logs.append(f"--- {role.upper()} GUEST ---\n" + serial.read_text( + encoding="utf-8", errors="replace")) + if result.returncode: + raise RuntimeError( + f"covered-root QEMU container exited with status {result.returncode}; see {stage}") + return container_output + "\n" + "\n".join(logs) + except RuntimeError: + inspected = _run(["docker", "container", "inspect", name]) + if inspected.returncode == 0: + _run(["docker", "stop", "--time", "10", name], timeout=30) + _run(["docker", "rm", name], timeout=30) + raise + + def _prepare_inputs(stage: Path, artifact: Path, wheelhouse: Path, requirements: Path, restic: Path, uv: Path) -> tuple[Path, dict[str, str]]: input_dir = stage / "inputs" @@ -746,6 +2039,9 @@ def _prepare_inputs(stage: Path, artifact: Path, wheelhouse: Path, requirements: def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scenario", choices=("timer-reboot", "covered-root"), + default="timer-reboot", + help="existing reboot drill or isolated covered-root replacement emulation") parser.add_argument("--artifact", type=Path, required=True, help="built candidate .whl") parser.add_argument("--wheel-sha256", "--artifact-sha256", dest="wheel_sha256", required=True, help="expected candidate wheel SHA-256") @@ -757,6 +2053,8 @@ def _parse_args() -> argparse.Namespace: parser.add_argument("--uv", type=Path, required=True, help="existing uv executable used to install locked wheels inside guest") parser.add_argument("--state-root", type=Path, default=DEFAULT_STATE, help="private ignored task state root") parser.add_argument("--timeout-seconds", type=int, default=1500, help="maximum guest runtime") + parser.add_argument("--accelerator", choices=("kvm", "tcg"), default="kvm", + help="KVM hardware virtualization or slower TCG software emulation") return parser.parse_args() @@ -780,42 +2078,130 @@ def main() -> int: os.chmod(stage, 0o700) artifact_copy = stage / "candidate.whl" _copy_regular(artifact, artifact_copy) - seed = _create_seed(stage, args.wheel_sha256, args.ubuntu_codename) + if args.scenario == "covered-root": + source_seed = _create_covered_root_seed( + stage, args.wheel_sha256, args.ubuntu_codename, "source") + replacement_seed = _create_covered_root_seed( + stage, args.wheel_sha256, args.ubuntu_codename, "replacement") + else: + seed = _create_seed(stage, args.wheel_sha256, args.ubuntu_codename) inputs_iso, input_digests = _prepare_inputs(stage, artifact, args.wheelhouse, args.requirements, restic, uv) _container_image() start = time.monotonic() - serial = _docker_qualification(stage, image, seed, inputs_iso, args.timeout_seconds) + if args.scenario == "covered-root": + serial = _docker_covered_root_qualification( + stage, image, source_seed, replacement_seed, inputs_iso, args.timeout_seconds, + accelerator=args.accelerator) + source_match = re.search(r"SOURCE_COMPLETE (\{.*\})", serial) + replacement_match = re.search(r"COVERED_ROOT_RESULT (\{.*\})", serial) + if source_match is None or replacement_match is None: + failure_marker = re.search(r"FAIL [^\r\n]+", serial) + reason = failure_marker.group(0) if failure_marker else "covered-root guest proof is incomplete" + raise RuntimeError(f"covered-root emulation incomplete: {reason}; artifacts: {stage}") + source = json.loads(source_match.group(1)) + evidence = json.loads(replacement_match.group(1)) + checks = evidence.get("checks") + if (source.get("candidate_wheel_sha256") != args.wheel_sha256 + or evidence.get("candidate_wheel_sha256") != args.wheel_sha256 + or source.get("source_machine_digest") != evidence.get("source_machine_digest") + or source.get("source_machine_digest") == evidence.get("replacement_machine_digest") + or not isinstance(checks, dict) or not checks or not all(checks.values()) + or evidence.get("capture_manifest_sha256") != source.get("capture_manifest_sha256") + or evidence.get("sqlite_export_sha256") != source.get("sqlite_export_sha256") + or evidence.get("baseline_digest") != source.get("baseline_digest") + or evidence.get("snapshot_id") != source.get("snapshot_id") + or evidence.get("fixture_logical_bytes") != source.get("fixture_logical_bytes") + or source.get("acl_applied") is not True + or evidence.get("acl_applied") is not True + or type(evidence.get("fixture_logical_bytes")) is not int + or not 0 < evidence["fixture_logical_bytes"] < 1024 * 1024): + raise RuntimeError(f"covered-root guest evidence failed validation; artifacts: {stage}") + if (not re.fullmatch(r"[0-9a-f]{64}", source.get("source_machine_digest", "")) + or not re.fullmatch(r"[0-9a-f]{64}", evidence.get("replacement_machine_digest", "")) + or not re.fullmatch(r"[0-9a-f]{64}", source.get("baseline_digest", "")) + or not re.fullmatch(r"[0-9a-f]{64}", source.get("candidate_digest", "")) + or not re.fullmatch(r"[0-9a-f]{64}", source.get("capture_manifest_sha256", "")) + or not re.fullmatch(r"[0-9a-f]{64}", source.get("sqlite_export_sha256", "")) + or not isinstance(source.get("snapshot_id"), str) or not source["snapshot_id"]): + raise RuntimeError(f"covered-root guest digests are malformed; artifacts: {stage}") + timer_invocation = source.get("root_timer_invocation") + if (not isinstance(timer_invocation, dict) + or timer_invocation.get("unit") != "bbackup-covered.service" + or timer_invocation.get("result") != "success" + or timer_invocation.get("exit_status") != 0 + or timer_invocation.get("start_observed") is not True + or timer_invocation.get("exit_observed") is not True): + raise RuntimeError(f"covered-root root timer invocation is incomplete; artifacts: {stage}") + evidence.update({"source_machine_digest": source["source_machine_digest"], + "baseline_digest": source["baseline_digest"], + "candidate_digest": source["candidate_digest"], + "snapshot_id": source["snapshot_id"], + "capture_manifest_sha256": source["capture_manifest_sha256"], + "sqlite_export_sha256": source["sqlite_export_sha256"], + "root_timer_invocation": source["root_timer_invocation"], + "acl_applied": source["acl_applied"]}) + evidence_files = ("source-serial.log", "replacement-serial.log", "apk-installed.txt", + "source-guest.qcow2", "replacement-guest.qcow2", "recovery.raw") + for filename in evidence_files: + path = stage / filename + if not path.is_file() or path.is_symlink(): + raise RuntimeError(f"covered-root emulation artifact is missing: {filename}") + else: + serial = _docker_qualification(stage, image, seed, inputs_iso, args.timeout_seconds, + accelerator=args.accelerator) + result_marker = re.search(r"QUALIFICATION_RESULT (\{.*\})", serial) + if result_marker is None: + failure_marker = re.search(r"FAIL [^\r\n]+", serial) + reason = failure_marker.group(0) if failure_marker else "guest did not emit final qualification evidence" + raise RuntimeError(f"systemd guest qualification incomplete: {reason}; artifacts: {stage}") + evidence = json.loads(result_marker.group(1)) + if evidence.get("candidate_sha256") != args.wheel_sha256 or not all(evidence.get("checks", {}).values()): + raise RuntimeError(f"guest evidence is incomplete or for a different wheel; artifacts: {stage}") + if len(set(evidence.get("boot_ids", []))) < 3: + raise RuntimeError(f"qualification did not record three guest boots; artifacts: {stage}") elapsed = round(time.monotonic() - start, 1) - result_marker = re.search(r"QUALIFICATION_RESULT (\{.*\})", serial) - if result_marker is None: - failure_marker = re.search(r"FAIL [^\r\n]+", serial) - reason = failure_marker.group(0) if failure_marker else "guest did not emit final qualification evidence" - raise RuntimeError(f"systemd guest qualification incomplete: {reason}; artifacts: {stage}") - evidence = json.loads(result_marker.group(1)) - if evidence.get("candidate_sha256") != args.wheel_sha256 or not all(evidence.get("checks", {}).values()): - raise RuntimeError(f"guest evidence is incomplete or for a different wheel; artifacts: {stage}") - if len(set(evidence.get("boot_ids", []))) < 3: - raise RuntimeError(f"qualification did not record three guest boots; artifacts: {stage}") private_evidence = { "schema_version": 1, "candidate": {"path_label": artifact.name, "sha256": args.wheel_sha256}, "guest_image": {"distribution": f"Ubuntu {args.ubuntu_codename}", "codename": args.ubuntu_codename, "sha256": _sha256(image), "signed_manifest": "Canonical gpgv valid"}, "qemu_toolchain": {"image": ALPINE_IMAGE, "packages": list(ALPINE_PACKAGES), + "accelerator": args.accelerator, "verified": "image digest and installed package versions checked at runtime"}, "input_digests": input_digests, "elapsed_seconds": elapsed, "guest": evidence, - "evidence_files": ["serial.log", "apk-installed.txt", "guest.qcow2", "qualification.json"], + "evidence_files": (list(evidence_files) if args.scenario == "covered-root" else + ["serial.log", "apk-installed.txt", "guest.qcow2", "qualification.json"]), } + if args.scenario == "covered-root": + source_instance_id = (stage / "covered-root-seed-source" / "meta-data").read_text( + encoding="ascii").splitlines()[0].split(": ", 1)[1] + replacement_instance_id = (stage / "covered-root-seed-replacement" / "meta-data").read_text( + encoding="ascii").splitlines()[0].split(": ", 1)[1] + if source_instance_id == replacement_instance_id: + raise RuntimeError("source and replacement cloud-init identities must differ") + private_evidence.update({"scenario": "covered-root", + "qualification_classification": "local-emulation-only", + "limitations": ["This is local VM emulation, not independent custody or a remote provider window."], + "guest_topology": {"source_overlay": "source-guest.qcow2", + "replacement_overlay": "replacement-guest.qcow2", + "source_backing_image_sha256": _sha256(image), + "replacement_backing_image_sha256": _sha256(image), + "recovery_disk": "recovery.raw", "recovery_disk_size_bytes": 256 * 1024 * 1024, + "source_cloud_init_instance_id": source_instance_id, + "replacement_cloud_init_instance_id": replacement_instance_id, + "separate_os_overlays": True, "shared_dedicated_recovery_disk": True}}) evidence_path = stage / "host-evidence.json" evidence_path.write_text(json.dumps(private_evidence, sort_keys=True, indent=2) + "\n", encoding="utf-8") evidence_path.chmod(0o600) - print(json.dumps({"success": True, "candidate_sha256": args.wheel_sha256, - "guest": evidence.get("os"), "checks": evidence["checks"], - "elapsed_seconds": elapsed, "private_evidence": str(evidence_path)}, - sort_keys=True)) + summary = {"success": True, "candidate_sha256": args.wheel_sha256, + "guest": evidence.get("os"), "checks": evidence["checks"], + "elapsed_seconds": elapsed, "private_evidence": str(evidence_path)} + if args.scenario == "covered-root": + summary.update({"scenario": "covered-root", "classification": "local-emulation-only"}) + print(json.dumps(summary, sort_keys=True)) return 0 diff --git a/tests/test_b2.py b/tests/test_b2.py index c0cd496..563aefe 100644 --- a/tests/test_b2.py +++ b/tests/test_b2.py @@ -4,17 +4,101 @@ import pytest -from bbackup.b2 import B2Error, inspect_b2, inspect_sdk +from bbackup.b2 import B2Error, B2VersionLockReader, inspect_b2, inspect_sdk from bbackup.operations import OperationRunner +def _native_lock_api(info=None, *, allowed=None, endpoint="https://s3.us-west-004.backblazeb2.com", bucket_id="bucket-id-1"): + auth = { + "capabilities": ["listBuckets", "readFiles", "readFileRetentions", "readFileLegalHolds"], + "buckets": [{"id": "bucket-id-1", "name": "backup-bucket"}], + "namePrefix": "restic/", + } + if allowed: + auth.update(allowed) + if info is None: + info = { + "bucketId": "bucket-id-1", "fileId": "file-v1", "fileName": "restic/data", "action": "upload", + "fileRetention": {"isClientAuthorizedToRead": True, "value": {"mode": None, "retainUntilTimestamp": None}}, + "legalHold": {"isClientAuthorizedToRead": True, "value": None}, + } + bucket = SimpleNamespace(name="backup-bucket", id_=bucket_id) + return SimpleNamespace( + account_info=SimpleNamespace( + get_allowed=lambda: auth, + get_s3_api_url=lambda: endpoint, + get_api_url=lambda: "https://api001.backblazeb2.com", + ), + list_buckets=lambda **kwargs: [bucket], + session=SimpleNamespace(get_file_info_by_id=lambda file_id: info), + ) + + +def test_native_lock_reader_exact_authorized_null_and_future_protection(): + from datetime import datetime, timezone + + now = datetime(2026, 10, 20, tzinfo=timezone.utc) + api_value = _native_lock_api() + reader = B2VersionLockReader( + "backup-bucket", "bucket-id-1", "restic/", "https://s3.us-west-004.backblazeb2.com", + api=api_value, access_key_id="key", secret_access_key="secret", + ) + evidence, held = reader.read("restic/data", "file-v1", now=now) + assert evidence["provider"] == "b2-native" and evidence["legal_hold"] == "OFF" + assert evidence["legal_hold_value"] is None + assert evidence["mode"] is None and evidence["retain_until_timestamp_ms"] is None + assert held is False + api_value.session.get_file_info_by_id = lambda _id: { + **_native_lock_api().session.get_file_info_by_id("file-v1"), + "fileRetention": {"isClientAuthorizedToRead": True, "value": { + "mode": "governance", "retainUntilTimestamp": int(now.timestamp() * 1000) + 600000, + }}, + } + assert reader.read("restic/data", "file-v1", now=now)[1] is True + + +@pytest.mark.parametrize("damage", [ + {"bucketId": "wrong"}, {"fileId": "wrong"}, {"fileName": "restic/other"}, {"action": "hide"}, + {"fileRetention": {"isClientAuthorizedToRead": False, "value": {"mode": None, "retainUntilTimestamp": None}}}, + {"legalHold": {"isClientAuthorizedToRead": False, "value": "off"}}, + {"fileRetention": {"isClientAuthorizedToRead": True, "value": {"mode": None, "retainUntilTimestamp": 0}}}, + {"legalHold": {"isClientAuthorizedToRead": True, "value": "unknown"}}, +]) +def test_native_lock_reader_rejects_ambiguous_exact_version(damage): + from datetime import datetime, timezone + + info = dict(_native_lock_api().session.get_file_info_by_id("file-v1")) + info.update(damage) + reader = B2VersionLockReader( + "backup-bucket", "bucket-id-1", "restic/", "https://s3.us-west-004.backblazeb2.com", + api=_native_lock_api(info), access_key_id="key", secret_access_key="secret", + ) + with pytest.raises(B2Error, match="unknown"): + reader.read("restic/data", "file-v1", now=datetime.now(timezone.utc)) + + +@pytest.mark.parametrize("kwargs", [ + {"bucket_id": "wrong"}, + {"endpoint": "https://s3.us-east-005.backblazeb2.com"}, + {"allowed": {"namePrefix": "restic/other/"}}, + {"allowed": {"capabilities": ["listBuckets", "readFiles"]}}, +]) +def test_native_lock_reader_rejects_scope_or_endpoint_mismatch(kwargs): + with pytest.raises(B2Error, match="differs"): + B2VersionLockReader( + "backup-bucket", "bucket-id-1", "restic/", "https://s3.us-west-004.backblazeb2.com", + api=_native_lock_api(**kwargs), access_key_id="key", secret_access_key="secret", + ) + + class FakeBucket: name = "backup-bucket" + id_ = "bucket-id-1" lifecycle_rules = [{"fileNamePrefix": "", "daysFromHidingToDeleting": 30}] def ls(self, **kwargs): assert kwargs == { - "path": "restic/", + "path": kwargs["path"], "latest_only": False, "recursive": True, "fetch_count": 1, @@ -22,13 +106,21 @@ def ls(self, **kwargs): yield object(), None -def api(capabilities=None, rules=None): - caps = capabilities or ["listBuckets", "listFiles", "readFiles", "writeFiles"] +def api(capabilities=None, rules=None, *, allowed=None, bucket_id="bucket-id-1"): + caps = ( + ["listBuckets", "listFiles", "readFiles", "writeFiles"] + if capabilities is None + else capabilities + ) + auth = {"capabilities": caps, "buckets": [{"id": "bucket-id-1", "name": "backup-bucket"}], "namePrefix": "restic/"} + if allowed is not None: + auth.update(allowed) info = SimpleNamespace( - get_allowed=lambda: {"capabilities": caps, "namePrefix": "restic/"}, + get_allowed=lambda: auth, get_s3_api_url=lambda: "https://s3.us-west-004.backblazeb2.com", ) bucket = FakeBucket() + bucket.id_ = bucket_id if rules is not None: bucket.lifecycle_rules = rules return SimpleNamespace(account_info=info, list_buckets=lambda **kwargs: [bucket]) @@ -40,6 +132,17 @@ def test_vendor_adapter_inspects_one_bucket_without_exposing_tokens(): assert result["observed_noncurrent_days"] == 30 assert result["credential_administration_denied"] assert result["protection_qualified"] is False + profile = result["restricted_source_profile"] + assert profile["version"] == 1 + assert profile["status"] == "eligible_from_observed_metadata" + assert profile["basis"] == "sdk_authorization_metadata_only" + assert profile["scope"]["key_bucket_matches_selected_bucket"] + assert profile["scope"]["requested_prefix_within_machine_prefix"] + assert profile["negative_test_proof"] == { + "status": "not_performed", + "permanent_deletion_denial_demonstrated": False, + "administrative_denial_demonstrated": False, + } assert not {"authorizationToken", "accountId", "applicationKey"} & result.keys() @@ -67,11 +170,91 @@ def test_overlapping_lifecycle_and_dangerous_key_are_reported(): assert result["protection_qualified"] is False -def test_inspection_rejects_missing_permissions_and_wrong_prefix(): +def test_observed_profile_allows_bucket_reads_but_requires_write_files(): + result = inspect_sdk( + "backup-bucket", + prefix="restic/", + api=api(["listBuckets", "listFiles", "readFiles", "readBuckets"]), + ) + profile = result["restricted_source_profile"] + assert result["version_listing_verified"] + assert profile["status"] == "unqualified" + assert profile["missing_required_capabilities"] == ["writeFiles"] + assert "required_capabilities_missing" in profile["reasons"] + assert "readBuckets" in profile["observed_capabilities"] + + +@pytest.mark.parametrize( + ("allowed", "bucket_id", "prefix", "expected_reason"), + [ + ({"buckets": None}, "bucket-id-1", "restic/", "key_not_bucket_bound"), + ({"buckets": [{"id": "other-bucket-id", "name": "backup-bucket"}]}, "bucket-id-1", "restic/", "key_bucket_id_mismatch"), + ({"buckets": [{"id": "bucket-id-1", "name": "backup-bucket"}, {"id": "second", "name": "another-bucket"}]}, "bucket-id-1", "restic/", "key_not_bucket_bound"), + ({}, None, "restic/", "selected_bucket_id_unavailable"), + ({"namePrefix": ""}, "bucket-id-1", "restic/", "machine_prefix_restriction_missing"), + ({"namePrefix": "restic/a/"}, "bucket-id-1", "restic/", "requested_prefix_outside_machine_prefix"), + ({"namePrefix": "restic/a/"}, "bucket-id-1", "restic/ab/", "requested_prefix_outside_machine_prefix"), + ({"namePrefix": "restic/a"}, "bucket-id-1", "restic/a/", "machine_prefix_not_canonical_directory"), + ], +) +def test_unqualified_bucket_or_directory_scope_is_reported(allowed, bucket_id, prefix, expected_reason): + auth = {"buckets": [{"id": "bucket-id-1", "name": "backup-bucket"}], "namePrefix": "restic/"} + auth.update(allowed) + result = inspect_sdk( + "backup-bucket", prefix=prefix, api=api(allowed=auth, bucket_id=bucket_id) + ) + profile = result["restricted_source_profile"] + assert profile["status"] == "unqualified" + assert expected_reason in profile["reasons"] + if expected_reason == "requested_prefix_outside_machine_prefix": + assert result["version_listing_verified"] is False + + +def test_administrative_and_unrecognized_capabilities_disqualify_observed_profile(): + result = inspect_sdk( + "backup-bucket", + prefix="restic/", + api=api( + ["listBuckets", "listFiles", "readFiles", "writeFiles", "writeKeys", "futurePermission"] + ), + ) + profile = result["restricted_source_profile"] + assert profile["status"] == "unqualified" + assert "administrative_capability_present" in profile["reasons"] + assert "unrecognized_capability_present" in profile["reasons"] + assert profile["negative_test_proof"]["administrative_denial_demonstrated"] is False + + +@pytest.mark.parametrize( + "auth", + [ + {"capabilities": ["listBuckets", 42]}, + {"buckets": "bucket-id-1"}, + {"buckets": [{"id": "bucket-id-1"}]}, + {"buckets": [None]}, + {"namePrefix": ["restic/"]}, + {"namePrefix": "restic/\0"}, + ], +) +def test_malformed_authorization_metadata_fails_closed_without_echoing_values(auth): + secretish = "authorizationToken=do-not-expose" + with pytest.raises(B2Error) as caught: + inspect_sdk( + "backup-bucket", + prefix="restic/", + api=api(allowed={**auth, "irrelevant": secretish}), + ) + assert "do-not-expose" not in str(caught.value) + + +def test_inspection_rejects_missing_read_access_and_reports_wrong_prefix(): with pytest.raises(B2Error): inspect_sdk("backup-bucket", prefix="restic/", api=api(["listFiles"])) - with pytest.raises(B2Error): - inspect_sdk("backup-bucket", prefix="different/", api=api()) + result = inspect_sdk("backup-bucket", prefix="different/", api=api()) + profile = result["restricted_source_profile"] + assert profile["status"] == "unqualified" + assert "requested_prefix_outside_machine_prefix" in profile["reasons"] + assert result["version_listing_verified"] is False def test_sdk_errors_are_sanitized(): @@ -86,6 +269,17 @@ def fail(**kwargs): assert "do-not-expose" not in str(caught.value) +def test_unrestricted_sdk_authorization_retains_read_only_listing(): + from b2sdk.v3 import AbstractAccountInfo + + defaults = AbstractAccountInfo.DEFAULT_ALLOWED + result = inspect_sdk("backup-bucket", prefix="restic/", api=api(allowed={ + "buckets": defaults["buckets"], "namePrefix": defaults["namePrefix"], + })) + assert result["version_listing_verified"] is True + assert result["restricted_source_profile"]["status"] == "unqualified" + + def test_real_child_missing_credentials_and_precancel(monkeypatch): monkeypatch.delenv("BBACKUP_TEST_B2_KEY_ID", raising=False) monkeypatch.delenv("BBACKUP_TEST_B2_KEY", raising=False) diff --git a/tests/test_capture.py b/tests/test_capture.py index 10735ee..a96d2c6 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -1,11 +1,13 @@ import sqlite3 import json +import hashlib from pathlib import Path from threading import Event import pytest -from bbackup.capture import CaptureError, capture_sources +from bbackup.capture import CaptureError, DurableCaptureArtifacts, capture_sources +from bbackup.linux_coverage import sqlite_file_identity from bbackup.models import Source @@ -35,6 +37,149 @@ def test_sqlite_transactional_export_and_cleanup(tmp_path): writer.close() +def test_covered_sqlite_manifest_binds_export_to_reviewed_source(tmp_path): + original = tmp_path / "database [one]*?.sqlite" + with sqlite3.connect(original) as database: + database.execute("CREATE TABLE committed (value TEXT)") + database.execute("INSERT INTO committed VALUES ('covered')") + context = {"root_id": "database", "path": str(original), "source_id": "db", + "identity": sqlite_file_identity(original), "recipe_digest": "a" * 64, + "baseline_digest": "b" * 64} + with capture_sources((Source("db", "sqlite", (str(original),)),), tmp_path, + coverage_sqlite={"db": context}) as paths: + exported = Path(paths[0]) + manifest = json.loads(Path(paths[-1]).read_text()) + assert manifest["schema_version"] == 2 + assert len(manifest["exports"]) == 1 + entry = manifest["exports"][0] + assert entry["coverage_root_id"] == "database" + assert entry["original_path"] == str(original) + assert entry["reviewed_identity"] == context["identity"] + assert entry["recipe_digest"] == "a" * 64 + assert entry["baseline_digest"] == "b" * 64 + assert entry["export_sha256"] == hashlib.sha256(exported.read_bytes()).hexdigest() + with sqlite3.connect(exported) as database: + assert database.execute("PRAGMA quick_check").fetchone() == ("ok",) + assert database.execute("SELECT value FROM committed").fetchall() == [("covered",)] + with pytest.raises(CaptureError, match="identity changed"): + original.chmod(0o600) + with capture_sources((Source("db", "sqlite", (str(original),)),), tmp_path, + coverage_sqlite={"db": context}): + pass + + +def test_sqlite_budget_preflight_refuses_export_before_writing(tmp_path): + from bbackup.resource_limits import ResourceLimitError + original = tmp_path / "live.sqlite" + with sqlite3.connect(original) as database: + database.execute("CREATE TABLE data(value TEXT)") + database.execute("INSERT INTO data VALUES ('committed')") + + class Monitor: + def check(self, upcoming=0): + if upcoming > 10: + raise ResourceLimitError("Observed free-space reserve fell below the configured threshold") + + with pytest.raises(ResourceLimitError, match="reserve"): + with capture_sources((Source("db", "sqlite", (str(original),)),), tmp_path, + resource_monitor=Monitor()): + pass + assert not list(tmp_path.glob("capture-*")) + + +def test_durable_first_export_survives_later_source_failure(tmp_path): + staging = tmp_path / "staging" + state = tmp_path / "state" + staging.mkdir(mode=0o700) + state.mkdir(mode=0o700) + original = tmp_path / "live.sqlite" + with sqlite3.connect(original) as database: + database.execute("CREATE TABLE data(value TEXT)") + database.execute("INSERT INTO data VALUES ('first point in time')") + artifacts = DurableCaptureArtifacts(staging, state, "a" * 64, "b" * 32, "daily") + with pytest.raises(CaptureError, match="existing database"): + with capture_sources((Source("first", "sqlite", (str(original),)), + Source("second", "sqlite", (str(tmp_path / "missing"),))), + staging, durable_artifacts=artifacts): + pytest.fail("later failed export became a backup") + record = json.loads(artifacts.record_path.read_text()) + assert record["phase"] == "prepared" + assert [item["name"] for item in record["exports"]] == ["first.sqlite"] + exported = artifacts.directory / "first.sqlite" + assert record["exports"][0]["sha256"] == hashlib.sha256(exported.read_bytes()).hexdigest() + with sqlite3.connect(original) as database: + database.execute("UPDATE data SET value='later state'") + with sqlite3.connect(exported) as database: + assert database.execute("SELECT value FROM data").fetchone() == ("first point in time",) + + +def test_durable_cleanup_rejects_tampered_record_path(tmp_path): + staging = tmp_path / "staging" + state = tmp_path / "state" + staging.mkdir(mode=0o700) + state.mkdir(mode=0o700) + artifacts = DurableCaptureArtifacts(staging, state, "a" * 64, "b" * 32, "daily") + artifacts.prepare() + exported = artifacts.directory / "db.dump" + artifacts.begin_export(exported) + exported.write_bytes(b"point in time") + exported.chmod(0o600) + artifacts.finish_export(exported) + manifest = artifacts.directory / "bbackup-capture.json" + artifacts.begin_export(manifest) + manifest.write_text("{}") + manifest.chmod(0o600) + artifacts.ready(manifest) + outside = staging / "foreign" + outside.write_bytes(b"keep") + record = json.loads(artifacts.record_path.read_text()) + record["phase"] = "cleanup_started" + record["exports"][0]["name"] = "../foreign" + artifacts.record_path.write_text(json.dumps(record)) + with pytest.raises(CaptureError, match="names"): + DurableCaptureArtifacts.load(staging, state, "a" * 64, "b" * 32).cleanup_success() + assert outside.read_bytes() == b"keep" + + +def test_reviewed_pending_export_is_hashed_and_change_blocks_discard(tmp_path): + staging = tmp_path / "staging" + state = tmp_path / "state" + staging.mkdir(mode=0o700) + state.mkdir(mode=0o700) + artifacts = DurableCaptureArtifacts(staging, state, "a" * 64, "b" * 32, "daily") + artifacts.prepare() + pending = artifacts.directory / "partial.dump" + artifacts.begin_export(pending) + pending.write_bytes(b"unfinished point in time") + pending.chmod(0o600) + artifacts.reviewed_failure("failed") + reviewed = json.loads(artifacts.record_path.read_text()) + assert reviewed["pending_observed"] == [{ + "name": "partial.dump", "size": pending.stat().st_size, + "sha256": hashlib.sha256(pending.read_bytes()).hexdigest()}] + digest = artifacts.record_sha256() + pending.write_bytes(b"changed but still retained") + with pytest.raises(CaptureError, match="changed"): + artifacts.discard(digest) + assert pending.exists() + + +def test_native_export_checks_budget_before_writing(): + import io + from bbackup.capture import _bounded_write + from bbackup.resource_limits import ResourceLimitError + + class Monitor: + def check(self, upcoming=0): + if upcoming > 4: + raise ResourceLimitError("Observed free-space reserve fell below the configured threshold") + + output = io.BytesIO() + with pytest.raises(ResourceLimitError, match="reserve"): + _bounded_write(output, b"database export", Monitor()) + assert output.getvalue() == b"" + + def test_files_not_copied(tmp_path): source = Source("files", "files", (str(tmp_path),)) with capture_sources((source,), tmp_path) as paths: diff --git a/tests/test_cloud_maintenance_service.py b/tests/test_cloud_maintenance_service.py new file mode 100644 index 0000000..f489ede --- /dev/null +++ b/tests/test_cloud_maintenance_service.py @@ -0,0 +1,198 @@ +"""Local Restic exercise of the central maintenance command boundary.""" + +import json +import shutil +import subprocess +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest + +from bbackup.models import Configuration, HostBindings, RepositoryBinding +from bbackup.operations import OperationRun, OperationStatus, repository_key +from bbackup.service import BackupService, ServiceError + + +def _restic(repo, password, *args): + return subprocess.run( + ["restic", "-r", str(repo), "--password-file", str(password), *args], + check=True, capture_output=True, text=True, + ).stdout + + +@pytest.mark.integration +@pytest.mark.skipif(shutil.which("restic") is None, reason="requires restic") +def test_real_local_restic_maintenance_forgets_only_exact_target_without_repack(tmp_path, monkeypatch): + repo, staging, data = (tmp_path / name for name in ("repository", "staging", "data")) + staging.mkdir(mode=0o700) + data.mkdir() + password = tmp_path / "password" + password.write_text("fixture password") + password.chmod(0o600) + _restic(repo, password, "init") + ids = [] + for value in ("protected", "obsolete", "unknown"): + (data / "value").write_text(value) + _restic(repo, password, "backup", str(data)) + ids.append(json.loads(_restic(repo, password, "snapshots", "--json"))[-1]["id"]) + config = Configuration.parse({"schema_version": 3, + "repositories": [{"name": "replica", "role": "replica"}], "sources": [], "jobs": []}) + service = BackupService(config, HostBindings(tmp_path / "state", { + "replica": RepositoryBinding(str(repo), password)}, schema_version=3)) + bindings = tmp_path / "maintenance-bindings.json" + bindings.write_text(json.dumps({"schema_version": 1, "staging_dir": str(staging), + "access_key_id_env": "TEST_MAINTENANCE_ID", "secret_access_key_env": "TEST_MAINTENANCE_KEY"})) + bindings.chmod(0o600) + monkeypatch.setenv("TEST_MAINTENANCE_ID", "separate-key") + monkeypatch.setenv("TEST_MAINTENANCE_KEY", "separate-secret") + monkeypatch.setenv("B2_APPLICATION_KEY", "must-not-enter-restic") + monkeypatch.setattr(service, "_maintenance_inspect_b2", lambda *_args: None) + + class Monitor: + def __init__(self, *_args, **_kwargs): + pass + def __enter__(self): + return self + def __exit__(self, *_args): + pass + def check(self): + pass + + monkeypatch.setattr("bbackup.resource_limits.ResourceMonitor", Monitor) + + class Session: + selection = SimpleNamespace(resources={"resource_budget": { + "schema_version": 1, "staging_max_bytes": 1048576, + "minimum_free_bytes": 65536, "cache_max_bytes": 0}, + "timeout_seconds": 120}) + def __init__(self): + self.phases = [] + self.plan_data = None + def plan(self, observed, *, bindings_sha256, credential_identity): + assert set(observed["visible_snapshot_ids"]) == set(ids) + assert credential_identity == {"key_id": "separate-key", "profile": "restricted"} + assert len(bindings_sha256) == 64 + self.plan_data = {"operation_id": "a" * 32, "target_snapshot_ids": [ids[1]], + "bindings_sha256": bindings_sha256} + return {"plan_id": "a" * 32, "plan_sha256": "b" * 64, + "target_snapshot_ids": [ids[1]]} + def validate_plan(self, _id, _sha, observed, **_kwargs): + assert set(observed["visible_snapshot_ids"]) == set(ids) + return self.plan_data + def record_phase(self, _plan, phase, status, *, evidence=None): + self.phases.append((phase, status, evidence)) + def verify_protected_versions(self): + return {"protected_version_sha256": "c" * 64, "protected_version_count": 1} + def finish_reconciliation(self, _plan, observations): + assert observations["visible_snapshot_ids"] == sorted([ids[0], ids[2]]) + return {"status": "complete", "outcome": "succeeded"} + + session = Session() + class Adapter: + _credential_pair = ("inspector", "inspector-secret") + endpoint_url = "https://s3.us-west-004.backblazeb2.com" + @contextmanager + def maintenance_session(self, *_args, **_kwargs): + yield session + + adapter = Adapter() + values = {"repository": "replica", "policy_file": str(tmp_path / "policy"), + "grant_file": str(tmp_path / "grant"), "maintenance_bindings_file": str(bindings), + "quiescent_window_confirmed": True} + observed_env = [] + original_run = service.runner.run + def run(plan, **kwargs): + observed_env.append((plan.argv, plan.env)) + return original_run(plan, **kwargs) + monkeypatch.setattr(service.runner, "run", run) + lease_id = repository_key(str(repo)) + preview = service.cloud_maintenance("cloud-maintenance plan", adapter, values, [], lease_id=lease_id) + assert preview["target_snapshot_ids"] == [ids[1]] + assert set(json.loads(_restic(repo, password, "snapshots", "--json"))[-1]) >= {"id"} + result = service.cloud_maintenance("cloud-maintenance apply", adapter, + {**values, **{key: preview[key] for key in ("plan_id", "plan_sha256")}}, + [], lease_id=lease_id) + assert result["status"] == "complete" + assert {item["id"] for item in json.loads(_restic(repo, password, "snapshots", "--json"))} == {ids[0], ids[2]} + for index, expected in ((0, "protected"), (2, "unknown")): + target = tmp_path / f"restored-{index}" + _restic(repo, password, "restore", ids[index], "--target", str(target)) + assert (target / str(data).lstrip("/") / "value").read_text() == expected + assert any("prune" in argv and ("--max-repack-size", "0") == argv[-2:] for argv, _env in observed_env) + assert any("forget" in argv and argv[-2:] == ("--", ids[1]) for argv, _env in observed_env) + assert all("B2_APPLICATION_KEY" not in env for _argv, env in observed_env) + assert all(env["AWS_ACCESS_KEY_ID"] == "separate-key" for _argv, env in observed_env) + assert [name for name, status, _ in session.phases if status == "completed"] == [ + "check_before", "forget", "verify_snapshots", "prune", "check_after", "protected_versions"] + + +def test_maintenance_bindings_reject_inline_or_reused_credentials(tmp_path, monkeypatch): + config = Configuration.parse({"schema_version": 3, "repositories": [ + {"name": "replica", "role": "replica"}], "sources": [], "jobs": []}) + password = tmp_path / "password" + password.write_text("fixture") + password.chmod(0o600) + service = BackupService(config, HostBindings(tmp_path / "state", { + "replica": RepositoryBinding(str(tmp_path / "repo"), password)}, schema_version=3)) + staging = tmp_path / "staging" + staging.mkdir(mode=0o700) + bindings = tmp_path / "bindings.json" + base = {"schema_version": 1, "staging_dir": str(staging), + "access_key_id_env": "TEST_MAINTENANCE_ID", "secret_access_key_env": "TEST_MAINTENANCE_KEY"} + monkeypatch.setenv("TEST_MAINTENANCE_ID", "adapter-key") + monkeypatch.setenv("TEST_MAINTENANCE_KEY", "adapter-secret") + bindings.write_text(json.dumps({**base, "secret_access_key": "inline-secret"})) + bindings.chmod(0o600) + adapter = SimpleNamespace(_credential_pair=("adapter-key", "adapter-secret"), + endpoint_url="https://s3.us-west-004.backblazeb2.com") + with pytest.raises(ServiceError, match="bindings are invalid"): + service._maintenance_bindings(bindings, adapter) + bindings.write_text(json.dumps(base)) + with pytest.raises(ServiceError, match="must differ"): + service._maintenance_bindings(bindings, adapter) + monkeypatch.setenv("TEST_MAINTENANCE_ID", "separate-key") + assert service._maintenance_bindings(bindings, adapter)[1]["profile"] == "restricted" + bindings.write_text(json.dumps({**base, "credential_profile": "management"})) + assert service._maintenance_bindings(bindings, adapter)[1]["profile"] == "management" + + +@pytest.mark.parametrize("profile,scope,accepted", [ + ("restricted", "unqualified", False), + ("restricted", "eligible_from_observed_metadata", True), + ("management", "unqualified", True), +]) +def test_b2_inspection_requires_restricted_default_or_explicit_management( + tmp_path, monkeypatch, profile, scope, accepted +): + config = Configuration.parse({"schema_version": 3, "repositories": [ + {"name": "replica", "role": "replica"}], "sources": [], "jobs": []}) + password = tmp_path / "password" + password.write_text("fixture") + password.chmod(0o600) + service = BackupService(config, HostBindings(tmp_path / "state", { + "replica": RepositoryBinding(str(tmp_path / "repo"), password)}, schema_version=3)) + adapter = SimpleNamespace(endpoint_url="https://s3.us-west-004.backblazeb2.com", + bucket="backup-bucket", prefix="repo/") + payload = json.dumps({"bucket": "backup-bucket", "prefix": "repo/", + "s3_endpoint": adapter.endpoint_url, "bucket_access_verified": True, + "version_listing_verified": True, "write_capability_present": True, + "read_capability_present": True, + "restricted_source_profile": {"status": scope}}).encode() + observed = [] + def run(plan, **_kwargs): + observed.append(plan) + return OperationRun(plan.operation_id, plan.repository_id, + OperationStatus.SUCCEEDED, 0, "2026-10-20T00:00:00Z", "2026-10-20T00:00:00Z", + 0.01, payload, b"", len(payload), 0) + monkeypatch.setattr(service.runner, "run", run) + environment = {"PATH": "/usr/bin", "HOME": str(tmp_path), "TMPDIR": str(tmp_path), + "AWS_ACCESS_KEY_ID": "source-key", "AWS_SECRET_ACCESS_KEY": "source-secret"} + if accepted: + service._maintenance_inspect_b2(adapter, "local-repo", environment, + ("source-key", "source-secret"), profile, None) + else: + with pytest.raises(ServiceError, match="unqualified"): + service._maintenance_inspect_b2(adapter, "local-repo", environment, + ("source-key", "source-secret"), profile, None) + assert "AWS_SECRET_ACCESS_KEY" not in observed[0].env + assert observed[0].env["B2_APPLICATION_KEY_ID"] == "source-key" diff --git a/tests/test_cloud_retention.py b/tests/test_cloud_retention.py new file mode 100644 index 0000000..00b1d14 --- /dev/null +++ b/tests/test_cloud_retention.py @@ -0,0 +1,1371 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +import uuid +from datetime import datetime as RealDateTime, timedelta, timezone +from pathlib import Path + +import pytest + +import bbackup.cloud as cloud +from bbackup.cloud import CloudError, S3CloudAdapter +from bbackup.b2 import B2VersionLockReader +from bbackup.operations import OperationLedger, OperationPlan, OperationStatus, RepositoryConflictError, repository_key +from bbackup.recovery import verify_checkpoint as real_verify_checkpoint, verify_recovery_acceptance as real_verify_acceptance + + +REPOSITORY_ID = "d" * 64 +VERIFIER = "F" * 40 +SOURCE_SIGNER = "E" * 40 +ENDPOINT = "https://retention.example.test" +BUCKET = "retention-fixture" +PREFIX = "repo/" + + +class FrozenDateTime(RealDateTime): + current = RealDateTime(2026, 10, 20, tzinfo=timezone.utc) + + @classmethod + def now(cls, tz=None): + value = cls.current + return value if tz is None else value.astimezone(tz) + + +class RetentionStore: + def __init__(self): + self.versions = [ + ("repo/data/restore.bin", "historical-v1", False, 8), + ("repo/data/restore.bin", "current-v2", True, 11), + ("repo/other/old.bin", "old-v1", False, 4), + ("repo/other/old.bin", "current-v2", True, 5), + ] + self.markers = [("repo/markers/removed.bin", "delete-v2", True)] + self.rules = [] + self.legal_holds = {} + self.retention = {} + self.lock_error = False + self.lifecycle_error = False + self.deleted = [] + self.delete_mode = "success" + self.bodies = {("repo/data/restore.bin", "historical-v1"): b"historic"} + self.request_id = 0 + + def _metadata(self): + self.request_id += 1 + return {"ResponseMetadata": {"RequestId": f"request-{self.request_id}"}} + + def get_bucket_versioning(self, **_kwargs): + return {"Status": "Enabled", "MFADelete": "Disabled", **self._metadata()} + + def get_bucket_lifecycle_configuration(self, **_kwargs): + if self.lifecycle_error: + raise RuntimeError("sensitive lifecycle service detail") + return {"Rules": json.loads(json.dumps(self.rules)), **self._metadata()} + + def get_object_lock_configuration(self, **_kwargs): + return { + "ObjectLockConfiguration": {"ObjectLockEnabled": "Enabled"}, + **self._metadata(), + } + + def list_object_versions(self, **kwargs): + prefix = kwargs["Prefix"] + versions = [ + {"Key": key, "VersionId": version, "IsLatest": latest, "Size": size, + "LastModified": RealDateTime(2001, 1, 1, tzinfo=timezone.utc)} + for key, version, latest, size in self.versions + if key.startswith(prefix) + ] + markers = [ + {"Key": key, "VersionId": version, "IsLatest": latest} + for key, version, latest in self.markers + if key.startswith(prefix) + ] + return {"Versions": versions, "DeleteMarkers": markers, "IsTruncated": False} + + def get_object_legal_hold(self, **kwargs): + if self.lock_error: + raise RuntimeError("sensitive lock detail") + key = (kwargs["Key"], kwargs["VersionId"]) + return {"LegalHold": {"Status": self.legal_holds.get(key, "OFF")}} + + def get_object_retention(self, **kwargs): + if self.lock_error: + raise RuntimeError("sensitive retention detail") + key = (kwargs["Key"], kwargs["VersionId"]) + return {"Retention": dict(self.retention.get(key, {}))} + + def delete_object(self, **kwargs): + assert set(kwargs) == {"Bucket", "Key", "VersionId"} + self.deleted.append((kwargs["Key"], kwargs["VersionId"])) + if self.delete_mode == "lose_before": + raise TimeoutError("sensitive connection detail") + self.versions = [ + row for row in self.versions + if (row[0], row[1]) != (kwargs["Key"], kwargs["VersionId"]) + ] + if self.delete_mode == "lose_after": + raise TimeoutError("sensitive connection detail") + return {"VersionId": kwargs["VersionId"]} + + def get_object(self, **kwargs): + import io + + return { + "VersionId": kwargs["VersionId"], + "Body": io.BytesIO(self.bodies[(kwargs["Key"], kwargs["VersionId"])]), + } + + +def _write_json(path: Path, value: dict): + path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n") + path.chmod(0o600) + + +def _make_manifest(path: Path, snapshot: str, objects: list[tuple[str, str, bytes]]): + captured = (FrozenDateTime.current - timedelta(days=3)).isoformat().replace("+00:00", "Z") + rows = [ + { + "type": "checkpoint", + "schema_version": 1, + "provider": "s3-compatible-unqualified", + "endpoint_url": ENDPOINT, + "bucket": BUCKET, + "prefix": PREFIX, + "captured_at": captured, + "expires_at": None, + "source_repository": "local", + "repository_id": REPOSITORY_ID, + "source_snapshot_id": snapshot, + "quiescent_window_confirmed": True, + "protection_qualified": False, + "copyright": cloud._COPYRIGHT, + "copyright_url": cloud._COPYRIGHT_URL, + "private_notice": cloud._PRIVATE_NOTICE, + } + ] + rows.extend( + {"type": "object", "key": key, "version_id": version, "size": len(body)} + for key, version, body in objects + ) + rows.append( + { + "type": "end", + "item_count": len(objects), + "object_count": len(objects), + "delete_marker_count": 0, + } + ) + data = b"".join(cloud._json_line(row) for row in rows) + path.write_bytes(data) + path.chmod(0o600) + return hashlib.sha256(data).hexdigest(), captured + + +def _make_record(root: Path, snapshot: str, objects: list[tuple[str, str, bytes]]): + root.mkdir(mode=0o700) + manifest = root / "checkpoint.jsonl" + manifest_digest, captured_at = _make_manifest(manifest, snapshot, objects) + verified_at = (FrozenDateTime.current - timedelta(days=1)).isoformat().replace("+00:00", "Z") + acceptance = { + "type": "recovery_acceptance", + "schema_version": 1, + "manifest_sha256": manifest_digest, + "source_signer": SOURCE_SIGNER, + "verifier": VERIFIER, + "source_repository": "local", + "repository_id": REPOSITORY_ID, + "snapshot_id": snapshot, + "captured_at": captured_at, + "expires_at": None, + "verified_at": verified_at, + "files_restored": len(objects), + "bytes_restored": sum(len(body) for _, _, body in objects), + "recovery_verified": True, + "protection_qualified": False, + "copyright": f"Copyright © {FrozenDateTime.current.year} Crux Experts LLC", + "copyright_url": cloud._COPYRIGHT_URL, + "private_notice": cloud._PRIVATE_NOTICE, + } + _write_json(root / "acceptance.json", acceptance) + (root / "acceptance.sig").write_bytes(b"fixture central signature") + (root / "checkpoint.sig").write_bytes(b"fixture source signature") + for name in ("acceptance.sig", "checkpoint.sig"): + (root / name).chmod(0o600) + return root, manifest, manifest_digest, acceptance + + +def _write_authority(root: Path, known_good_digest: str, *, max_objects=20, grant_expiry=None): + state = root / "state" + state.mkdir(mode=0o700) + policy_path = root / "policy.json" + policy = { + "schema_version": 1, + "policy_id": "retention-test", + "catalog_id": "c" * 64, + "revision": 1, + "endpoint_url": ENDPOINT, + "bucket": BUCKET, + "prefix": PREFIX, + "repository_id": REPOSITORY_ID, + "trusted_verifier": VERIFIER, + "retention_counts": {"daily": 1, "weekly": 0, "monthly": 0}, + "grace_seconds": 30 * 24 * 60 * 60, + "plan_max_age_seconds": 90 * 24 * 60 * 60, + "max_apply_objects": max_objects, + "max_apply_bytes": 1024, + "state_dir": str(state), + "known_good_manifest_digests": [known_good_digest], + "pending_manifest_digests": [], + } + _write_json(policy_path, policy) + grant_path = root / "grant.json" + expires = grant_expiry or FrozenDateTime.current + timedelta(days=90) + grant = { + "schema_version": 1, + "policy_id": policy["policy_id"], + "catalog_id": policy["catalog_id"], + "policy_revision": policy["revision"], + "policy_sha256": cloud._retention_digest(policy_path.read_bytes()), + "endpoint_url": ENDPOINT, + "bucket": BUCKET, + "prefix": PREFIX, + "repository_id": REPOSITORY_ID, + "action": "delete_noncurrent_versions", + "expires_at": expires.isoformat().replace("+00:00", "Z"), + "max_apply_objects": max_objects, + "max_apply_bytes": 1024, + "stop_conditions": list(cloud._RETENTION_STOP_CONDITIONS), + } + _write_json(grant_path, grant) + return policy_path, grant_path, policy, grant, state + + +@pytest.fixture +def retention_env(tmp_path, monkeypatch): + FrozenDateTime.current = RealDateTime(2026, 10, 20, tzinfo=timezone.utc) + monkeypatch.setattr(cloud, "datetime", FrozenDateTime) + + from bbackup import recovery + + def verify_acceptance(record_dir, *, trusted_verifier, repository_id, snapshot, **_kwargs): + record = json.loads((Path(record_dir) / "acceptance.json").read_text()) + assert (Path(record_dir) / "acceptance.sig").is_file() + assert record["verifier"] == trusted_verifier + assert record["repository_id"] == repository_id + assert record["snapshot_id"] == snapshot + return record + + def verify_checkpoint(manifest, _signature, _signer, **_kwargs): + return hashlib.sha256(Path(manifest).read_bytes()).hexdigest() + + monkeypatch.setattr(recovery, "verify_recovery_acceptance", verify_acceptance) + monkeypatch.setattr(recovery, "verify_checkpoint", verify_checkpoint) + + store = RetentionStore() + adapter = S3CloudAdapter(BUCKET, client=store, endpoint_url=ENDPOINT, prefix="repo") + record_dir, manifest, manifest_digest, acceptance = _make_record( + tmp_path / "accepted", "a" * 64, + [("repo/data/restore.bin", "historical-v1", b"historic")], + ) + policy_path, grant_path, policy, grant, state = _write_authority(tmp_path, manifest_digest) + adapter.initialize_central_retention_catalog(policy_path, grant_path) + ledger = OperationLedger(state / "operations.sqlite3") + return { + "root": tmp_path, + "store": store, + "adapter": adapter, + "record_dir": record_dir, + "manifest": manifest, + "manifest_digest": manifest_digest, + "acceptance": acceptance, + "policy_path": policy_path, + "grant_path": grant_path, + "policy": policy, + "grant": grant, + "state": state, + "ledger": ledger, + "repository_lease_id": repository_key("fixture-repository-binding"), + } + + +def _plan(env, record_dirs=None): + return env["adapter"].plan_central_retention( + env["policy_path"], + env["grant_path"], + record_dirs if record_dirs is not None else [env["record_dir"]], + quiescent_window_confirmed=True, + repository_lease_id=env["repository_lease_id"], + ) + + +def _seed_and_age(env): + first = _plan(env) + assert first["candidate_count"] == 0 + catalog = json.loads((env["state"] / "retention-catalog.json").read_text()) + old_token = cloud._retention_version_token("repo/other/old.bin", "old-v1") + assert catalog["first_observed_noncurrent"][old_token] == cloud._iso(FrozenDateTime.current) + FrozenDateTime.current += timedelta(days=31) + return _plan(env) + + +def _apply(env, plan, *, mutation_client=None, cancel_event=None): + return env["adapter"].apply_central_retention( + plan["operation_id"], + plan["plan_sha256"], + env["policy_path"], + env["grant_path"], + [env["record_dir"]], + quiescent_window_confirmed=True, + ledger=env["ledger"], + cancel_event=cancel_event, + mutation_client=mutation_client or env["store"], + ) + + +def _refresh_grant_for_policy(env): + env["grant"]["policy_sha256"] = cloud._retention_digest(env["policy_path"].read_bytes()) + _write_json(env["grant_path"], env["grant"]) + + +def _prepare_maintenance(env, *, max_objects=20, max_bytes=1024): + second_dir, _manifest, _digest, _acceptance = _make_record( + env["root"] / "maintenance-old-acceptance", + "b" * 64, + [("repo/other/old.bin", "old-v1", b"old!")], + ) + env["second_record_dir"] = second_dir + records = [env["record_dir"], second_dir] + _plan(env, records) + FrozenDateTime.current += timedelta(days=31) + + policy = dict(env["policy"]) + policy["retention_counts"] = {"daily": 0, "weekly": 0, "monthly": 0} + policy["max_apply_objects"] = max_objects + policy["max_apply_bytes"] = max_bytes + _write_json(env["policy_path"], policy) + grant = dict(env["grant"]) + grant["max_apply_objects"] = max_objects + grant["max_apply_bytes"] = max_bytes + grant["policy_sha256"] = cloud._retention_digest(env["policy_path"].read_bytes()) + _write_json(env["grant_path"], grant) + env["policy"] = policy + env["grant"] = grant + _plan(env, records) + + policy = dict(env["policy"]) + policy["schema_version"] = 3 + policy["maintenance"] = { + "mode": "forget-and-prune-no-repack", + "resource_budget": { + "schema_version": 1, + "staging_max_bytes": 1024**2, + "minimum_free_bytes": 64 * 1024, + "cache_max_bytes": 0, + }, + "timeout_seconds": 3600, + } + _write_json(env["policy_path"], policy) + grant = dict(env["grant"]) + grant["schema_version"] = 2 + grant["action"] = "maintain_current_repository" + grant["policy_sha256"] = cloud._retention_digest(env["policy_path"].read_bytes()) + _write_json(env["grant_path"], grant) + env["policy"] = policy + env["grant"] = grant + return records + + +def _restic_evidence(snapshot_ids): + visible = sorted(snapshot_ids) + return { + "config_repository_id": REPOSITORY_ID, + "visible_snapshot_ids": visible, + "snapshot_inventory_sha256": cloud._maintenance_snapshot_digest(REPOSITORY_ID, visible), + } + + +def _maintenance_context(env, records, *, snapshot_ids=None): + return env["adapter"].maintenance_session( + env["policy_path"], + env["grant_path"], + records, + quiescent_window_confirmed=True, + repository_lease_id=env["repository_lease_id"], + ) + + +def _maintenance_plan(session, snapshot_ids=None): + evidence = _restic_evidence( + snapshot_ids if snapshot_ids is not None else ["a" * 64, "b" * 64, "c" * 64] + ) + public = session.plan( + evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + private = session.validate_plan( + public["plan_id"], + public["plan_sha256"], + evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + return evidence, public, private + + +def _assert_repository_reacquirable(env): + plan = OperationPlan( + operation_id=uuid.uuid4().hex, + repository_id=env["repository_lease_id"], + label="probe", + argv=("probe",), + destructive=False, + ) + lease = env["ledger"].begin(plan) + lease.finish(OperationStatus.SUCCEEDED, 0) + lease.close() + + +def _native_reader_for_retention(env): + """An exact metadata fake; raw SDK parsing is covered in test_b2.""" + reader = object.__new__(B2VersionLockReader) + reader.bucket = BUCKET + reader.bucket_id = "bucket-id-1" + reader.prefix = PREFIX + reader.endpoint_url = ENDPOINT + reader._credential_pair = ("key", "secret") + reader.locks = {} + + def read(key, file_id, *, now): + state = reader.locks.get((key, file_id), "none") + if state == "error": + raise RuntimeError("sensitive native provider detail") + evidence = { + "provider": "b2-native", "bucket": BUCKET, "bucket_id": reader.bucket_id, + "key": key, "file_id": file_id, "action": "upload", "legal_hold": "ON" if state == "hold" else "OFF", + "legal_hold_value": "on" if state == "hold" else None, + "mode": "governance" if state == "future" else None, + "retain_until_timestamp_ms": int((now + timedelta(minutes=10)).timestamp() * 1000) if state == "future" else None, + } + return evidence, state in {"hold", "future"} + + reader.read = read + env["adapter"].native_lock_reader = reader + env["adapter"]._credential_pair = ("key", "secret") + env["policy"].update({"schema_version": 2, "lock_metadata": {"provider": "b2-native", "bucket_id": reader.bucket_id}}) + _write_json(env["policy_path"], env["policy"]) + _refresh_grant_for_policy(env) + return reader + + +def test_native_schema2_null_can_age_and_lock_change_stops_predelete(retention_env): + env = retention_env + reader = _native_reader_for_retention(env) + plan = _seed_and_age(env) + assert plan["candidate_count"] == 1 + candidate = ("repo/other/old.bin", "old-v1") + reader.locks[candidate] = "hold" + with pytest.raises(CloudError): + _apply(env, plan) + assert env["store"].deleted == [] + reader.locks[candidate] = "error" + with pytest.raises(CloudError, match="native per-version lock evidence is unknown"): + cloud._CentralRetention(env["adapter"])._version_lock( + *candidate, lock_enabled=False, now=FrozenDateTime.current, + ) + assert env["store"].deleted == [] + + +@pytest.mark.parametrize("state", ["hold", "future"]) +def test_native_schema2_protected_versions_are_not_candidates(retention_env, state): + env = retention_env + reader = _native_reader_for_retention(env) + _plan(env) + FrozenDateTime.current += timedelta(days=31) + reader.locks[("repo/other/old.bin", "old-v1")] = state + assert _plan(env)["candidate_count"] == 0 + + +def test_native_schema2_strict_policy_binding_and_schema1_compatibility(retention_env): + env = retention_env + assert cloud._validate_retention_policy(env["policy"], env["adapter"])[0]["schema_version"] == 1 + reader = _native_reader_for_retention(env) + assert cloud._validate_retention_policy(env["policy"], env["adapter"])[0]["schema_version"] == 2 + for metadata in ( + {"provider": "b2-native", "bucket_id": "wrong"}, + {"provider": "s3", "bucket_id": reader.bucket_id}, + {"provider": "b2-native", "bucket_id": reader.bucket_id, "extra": True}, + ): + with pytest.raises(CloudError, match="Native lock reader"): + cloud._validate_retention_policy({**env["policy"], "lock_metadata": metadata}, env["adapter"]) + with pytest.raises(CloudError, match="Schema-1"): + cloud._validate_retention_policy({key: value for key, value in env["policy"].items() if key != "lock_metadata"} | {"schema_version": 1}, env["adapter"]) + with pytest.raises(ValueError, match="credentials differ"): + S3CloudAdapter( + BUCKET, client=env["store"], endpoint_url=ENDPOINT, prefix="repo", + native_lock_reader=reader, credential_pair=("other", "secret"), + ) + + +def test_schema1_generic_s3_lock_404_is_still_unknown(retention_env): + env = retention_env + class Ambiguous404(Exception): + response = {"Error": {"Code": "404"}} + + def unknown(**_kwargs): + raise Ambiguous404() + + env["store"].get_object_legal_hold = unknown + with pytest.raises(CloudError, match="lock evidence is unknown"): + cloud._CentralRetention(env["adapter"])._version_lock( + "repo/other/old.bin", "old-v1", lock_enabled=True, + now=FrozenDateTime.current, + ) + + +def test_schema2_single_attempt_delete_client_uses_frozen_pair(retention_env, monkeypatch): + import boto3 + + env = retention_env + _native_reader_for_retention(env) + calls = [] + expected = object() + monkeypatch.setattr(boto3, "client", lambda *args, **kwargs: calls.append((args, kwargs)) or expected) + assert cloud._CentralRetention(env["adapter"])._single_attempt_mutation_client() is expected + assert calls[0][0] == ("s3",) + options = calls[0][1] + assert options["aws_access_key_id"] == "key" + assert options["aws_secret_access_key"] == "secret" + assert options["config"].retries["total_max_attempts"] == 1 + + +def test_plan_protects_historical_restore_latest_versions_and_delete_markers(retention_env): + env = retention_env + plan = _seed_and_age(env) + plan_file = env["state"] / "retention-plans" / f"{plan['operation_id']}.json" + private_plan = json.loads(plan_file.read_text()) + assert [(item["key"], item["version_id"]) for item in private_plan["candidates"]] == [ + ("repo/other/old.bin", "old-v1") + ] + assert plan_file.stat().st_mode & 0o777 == 0o400 + + result = _apply(env, plan) + + assert result["status"] == "complete" + assert env["store"].deleted == [("repo/other/old.bin", "old-v1")] + assert ("repo/data/restore.bin", "historical-v1", False, 8) in env["store"].versions + assert ("repo/data/restore.bin", "current-v2", True, 11) in env["store"].versions + assert ("repo/other/old.bin", "current-v2", True, 5) in env["store"].versions + assert env["store"].markers == [("repo/markers/removed.bin", "delete-v2", True)] + + restored = env["root"] / "restored" + restored.mkdir() + result = env["adapter"].restore_checkpoint( + env["manifest"], restored, + expected_manifest_sha256=env["manifest_digest"], + expected_source_repository="local", + expected_repository_id=REPOSITORY_ID, + expected_source_snapshot_id="a" * 64, + ) + assert result.files_restored == 1 + assert (restored / "data" / "restore.bin").read_bytes() == b"historic" + + +def test_recently_hidden_old_upload_starts_grace_now_and_last_modified_is_ignored(retention_env): + env = retention_env + plan = _plan(env) + assert plan["candidate_count"] == 0 + catalog = json.loads((env["state"] / "retention-catalog.json").read_text()) + token = cloud._retention_version_token("repo/other/old.bin", "old-v1") + assert catalog["first_observed_noncurrent"][token] == cloud._iso(FrozenDateTime.current) + assert env["store"].deleted == [] + + +def test_missing_catalog_record_and_reaccepted_timestamp_fail_closed(retention_env): + env = retention_env + second_dir, _second_manifest, _second_digest, _second_acceptance = _make_record( + env["root"] / "second-accepted", "b" * 64, + [("repo/other/old.bin", "old-v1", b"old!")], + ) + _plan(env, [env["record_dir"], second_dir]) + with pytest.raises(CloudError, match="previously accepted"): + _plan(env) + + acceptance = json.loads((env["record_dir"] / "acceptance.json").read_text()) + acceptance["verified_at"] = (FrozenDateTime.current - timedelta(hours=2)).isoformat().replace("+00:00", "Z") + _write_json(env["record_dir"] / "acceptance.json", acceptance) + with pytest.raises(CloudError, match="replaced or re-accepted"): + _plan(env, [env["record_dir"], second_dir]) + + +@pytest.mark.parametrize("drift", ["policy", "grant", "inventory", "lifecycle", "lock"]) +def test_apply_rejects_policy_grant_inventory_lifecycle_and_lock_drift(retention_env, drift): + env = retention_env + plan = _seed_and_age(env) + if drift == "policy": + policy = dict(env["policy"]) + policy["revision"] += 1 + _write_json(env["policy_path"], policy) + elif drift == "grant": + grant = dict(env["grant"]) + grant["expires_at"] = (FrozenDateTime.current + timedelta(days=80)).isoformat().replace("+00:00", "Z") + _write_json(env["grant_path"], grant) + elif drift == "inventory": + env["store"].versions.append(("repo/other/new.bin", "new-v1", True, 2)) + elif drift == "lifecycle": + env["store"].rules = [{"ID": "new-rule", "Status": "Disabled", "Expiration": {"Days": 1}}] + elif drift == "lock": + env["store"].legal_holds[("repo/other/old.bin", "old-v1")] = "ON" + + with pytest.raises(CloudError): + _apply(env, plan) + assert env["store"].deleted == [] + + +def test_unknown_lock_and_enabled_expiration_rules_block_planning(retention_env): + env = retention_env + _plan(env) + FrozenDateTime.current += timedelta(days=31) + env["store"].lock_error = True + with pytest.raises(CloudError, match="lock evidence is unknown") as caught: + _plan(env) + assert "sensitive" not in str(caught.value) + + env["store"].lock_error = False + env["store"].rules = [{"ID": "expire", "Status": "Enabled", "Expiration": {"Days": 1}}] + with pytest.raises(CloudError, match="expiration blocks"): + _plan(env) + + +def test_expired_and_wrong_target_grants_are_rejected(retention_env): + env = retention_env + grant = dict(env["grant"]) + grant["expires_at"] = (FrozenDateTime.current - timedelta(seconds=1)).isoformat().replace("+00:00", "Z") + _write_json(env["grant_path"], grant) + with pytest.raises(CloudError, match="expired"): + _plan(env) + + grant = dict(env["grant"]) + grant["bucket"] = "different-bucket" + _write_json(env["grant_path"], grant) + with pytest.raises(CloudError, match="does not match"): + _plan(env) + + +def test_apply_bounds_reject_candidate_set_that_exceeds_authorized_limit(retention_env): + env = retention_env + env["store"].versions.extend( + [ + ("repo/other2/old.bin", "old-v1", False, 4), + ("repo/other2/old.bin", "current-v2", True, 5), + ] + ) + policy = dict(env["policy"]) + policy["max_apply_objects"] = 1 + _write_json(env["policy_path"], policy) + grant = dict(env["grant"]) + grant["max_apply_objects"] = 1 + grant["policy_sha256"] = cloud._retention_digest(env["policy_path"].read_bytes()) + _write_json(env["grant_path"], grant) + env["policy"] = policy + env["grant"] = grant + _plan(env) + FrozenDateTime.current += timedelta(days=31) + with pytest.raises(CloudError, match="authorized apply bounds"): + _plan(env) + assert env["store"].deleted == [] + + +def test_expired_grant_after_plan_prevents_any_mutation(retention_env): + env = retention_env + grant = dict(env["grant"]) + grant["expires_at"] = (FrozenDateTime.current + timedelta(days=45)).isoformat().replace("+00:00", "Z") + grant["policy_sha256"] = cloud._retention_digest(env["policy_path"].read_bytes()) + _write_json(env["grant_path"], grant) + env["grant"] = grant + _plan(env) + FrozenDateTime.current += timedelta(days=31) + plan = _plan(env) + FrozenDateTime.current += timedelta(days=15) + with pytest.raises(CloudError, match="expired"): + _apply(env, plan) + assert env["store"].deleted == [] + + +def test_lost_delete_response_is_reconciled_read_only_and_never_replayed(retention_env): + env = retention_env + plan = _seed_and_age(env) + env["store"].delete_mode = "lose_before" + result = _apply(env, plan) + assert result["status"] == "uncertain" + assert len(env["store"].deleted) == 1 + assert ("repo/other/old.bin", "old-v1", False, 4) in env["store"].versions + probe_fd = env["ledger"]._lock(env["repository_lease_id"]) + env["ledger"]._unlock(probe_fd) + + plan_path = env["state"] / "retention-plans" / f"{plan['operation_id']}.json" + reconciled = env["adapter"].reconcile_central_retention(plan_path, ledger=env["ledger"]) + assert reconciled == { + "status": "reconciled_abandoned", + "operation_id": plan["operation_id"], + "absent_count": 0, + "present_count": 1, + "unknown_count": 0, + } + with pytest.raises(CloudError, match="already applied"): + _apply(env, plan) + assert len(env["store"].deleted) == 1 + + +def test_lost_response_after_exact_delete_reconciles_observed_absence(retention_env): + env = retention_env + plan = _seed_and_age(env) + env["store"].delete_mode = "lose_after" + result = _apply(env, plan) + assert result["status"] == "uncertain" + assert not any(key == "repo/other/old.bin" and version == "old-v1" for key, version, *_ in env["store"].versions) + + plan_path = env["state"] / "retention-plans" / f"{plan['operation_id']}.json" + reconciled = env["adapter"].reconcile_central_retention(plan_path, ledger=env["ledger"]) + assert reconciled["status"] == "reconciled_succeeded" + assert reconciled["absent_count"] == 1 + assert len(env["store"].deleted) == 1 + + +def test_catalog_loss_cannot_reset_accepted_history(retention_env): + env = retention_env + _plan(env) + (env["state"] / "retention-catalog.json").unlink() + with pytest.raises(CloudError, match="catalog is missing"): + _plan(env) + with pytest.raises(CloudError, match="history exists"): + env["adapter"].initialize_central_retention_catalog(env["policy_path"], env["grant_path"]) + + +def test_known_good_pin_required_even_with_period_counts(retention_env): + env = retention_env + env["policy"]["known_good_manifest_digests"] = [] + _write_json(env["policy_path"], env["policy"]) + _refresh_grant_for_policy(env) + with pytest.raises(CloudError, match="known-good"): + _plan(env) + + +def test_schema3_maintenance_requires_schema2_maintenance_grant_and_budget(retention_env): + env = retention_env + records = _prepare_maintenance(env) + + grant = dict(env["grant"]) + grant["schema_version"] = 1 + _write_json(env["grant_path"], grant) + with pytest.raises(CloudError, match="authorize deletion only"): + with _maintenance_context(env, records): + pytest.fail("schema-1 grant must not open maintenance") + + grant = dict(env["grant"]) + grant["schema_version"] = 2 + grant["action"] = "delete_noncurrent_versions" + _write_json(env["grant_path"], grant) + with pytest.raises(CloudError, match="does not match"): + with _maintenance_context(env, records): + pytest.fail("deletion grant must not open maintenance") + + policy = dict(env["policy"]) + policy["maintenance"] = dict(policy["maintenance"]) + policy["maintenance"]["resource_budget"] = dict(policy["maintenance"]["resource_budget"]) + policy["maintenance"]["resource_budget"]["extra"] = 1 + _write_json(env["policy_path"], policy) + grant["action"] = "maintain_current_repository" + grant["policy_sha256"] = cloud._retention_digest(env["policy_path"].read_bytes()) + _write_json(env["grant_path"], grant) + with pytest.raises(CloudError, match="resource budget"): + with _maintenance_context(env, records): + pytest.fail("unknown resource-budget fields must be rejected") + + +def test_maintenance_selects_only_visible_accepted_unprotected_snapshots_and_preserves_catalog(retention_env): + env = retention_env + records = _prepare_maintenance(env) + catalog_path = env["state"] / "retention-catalog.json" + catalog_before = catalog_path.read_bytes() + evidence = _restic_evidence(["a" * 64, "b" * 64, "c" * 64]) + + with _maintenance_context(env, records) as session: + assert session.selection.accepted_snapshot_ids == ("a" * 64, "b" * 64) + assert session.selection.protected_snapshot_ids == ("a" * 64,) + assert session.selection.resources["current_payload_count"] == 2 + assert session.selection.resources["current_payload_bytes"] == 16 + with pytest.raises(TypeError): + session.selection.resources["timeout_seconds"] = 1 + + public, private = _maintenance_plan(session)[1:] + assert public["accepted_snapshot_ids"] == ["a" * 64, "b" * 64] + assert public["protected_snapshot_ids"] == ["a" * 64] + assert public["target_snapshot_ids"] == ["b" * 64] + assert private["target_snapshot_ids"] == ["b" * 64] + assert private["visible_snapshot_ids"] == evidence["visible_snapshot_ids"] + assert "credential_identity_sha256" in private + assert "key_id" not in json.dumps(private) + assert catalog_path.read_bytes() == catalog_before + + +@pytest.mark.parametrize("limit", ["objects", "bytes"]) +def test_maintenance_bounds_all_current_payload_inventory_not_prune_candidates(retention_env, limit): + env = retention_env + max_objects = 1 if limit == "objects" else 20 + max_bytes = 15 if limit == "bytes" else 1024 + records = _prepare_maintenance(env, max_objects=max_objects, max_bytes=max_bytes) + with pytest.raises(CloudError, match="current payload inventory exceeds authorized bounds"): + with _maintenance_context(env, records): + pytest.fail("all current object versions must fit within authorized bounds") + + +def test_maintenance_counts_held_current_payload_toward_quota(retention_env): + env = retention_env + records = _prepare_maintenance(env) + env["store"].legal_holds[("repo/data/restore.bin", "current-v2")] = "ON" + env["policy"]["max_apply_bytes"] = 5 + env["grant"]["max_apply_bytes"] = 5 + _write_json(env["policy_path"], env["policy"]) + _refresh_grant_for_policy(env) + _write_json(env["grant_path"], env["grant"]) + with pytest.raises(CloudError, match="current payload inventory exceeds authorized bounds"): + with _maintenance_context(env, records): + pytest.fail("held current payload must count toward a logical mutation bound") + + +def test_maintenance_only_catalog_and_records_enrollment_does_not_seed_version_age(retention_env): + env = retention_env + state = env["root"] / "maintenance-only-state" + state.mkdir(mode=0o700) + policy = dict(env["policy"]) + policy["schema_version"] = 3 + policy["state_dir"] = str(state) + policy["maintenance"] = {"mode": "forget-and-prune-no-repack", + "resource_budget": {"schema_version": 1, "staging_max_bytes": 1048576, + "minimum_free_bytes": 65536, "cache_max_bytes": 0}, + "timeout_seconds": 3600} + _write_json(env["policy_path"], policy) + grant = dict(env["grant"]) + grant["schema_version"] = 2 + grant["action"] = "maintain_current_repository" + grant["policy_sha256"] = cloud._retention_digest(env["policy_path"].read_bytes()) + _write_json(env["grant_path"], grant) + enrolled = env["adapter"].initialize_central_retention_catalog( + env["policy_path"], env["grant_path"], + expected_action="maintain_current_repository") + assert enrolled["status"] == "enrolled" + added = env["adapter"].enroll_central_maintenance_records( + env["policy_path"], env["grant_path"], [env["record_dir"]]) + assert added["accepted_record_count"] == 1 + catalog_path = state / "retention-catalog.json" + assert json.loads(catalog_path.read_text())["first_observed_noncurrent"] == {} + original = catalog_path.read_bytes() + assert env["adapter"].enroll_central_maintenance_records( + env["policy_path"], env["grant_path"], [env["record_dir"]])["status"] == "unchanged" + assert catalog_path.read_bytes() == original + with env["adapter"].maintenance_session( + env["policy_path"], env["grant_path"], [env["record_dir"]], + quiescent_window_confirmed=True, repository_lease_id=env["repository_lease_id"] + ) as session: + assert session.selection.accepted_snapshot_ids == ("a" * 64,) + + +def test_maintenance_inventory_ignores_lock_prefix_and_noncurrent_history_but_binds_dependencies(retention_env): + env = retention_env + records = _prepare_maintenance(env) + evidence = _restic_evidence(["a" * 64, "b" * 64, "c" * 64]) + + with _maintenance_context(env, records) as session: + public = session.plan( + evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + private = session.validate_plan( + public["plan_id"], public["plan_sha256"], evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + initial_inventory = private["inventory_sha256"] + initial_resources = private["resources"] + + env["store"].versions.extend( + [ + ("repo/data/restore.bin", "old-history-v0", False, 99999), + ("repo/locks/session.lock", "ephemeral-lock-v1", True, 99999), + ] + ) + validated = session.validate_plan( + public["plan_id"], public["plan_sha256"], evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + assert validated["inventory_sha256"] == initial_inventory + assert validated["resources"] == initial_resources + + env["store"].versions.append(("repo/dependencies/index", "dependency-v1", True, 3)) + with pytest.raises(CloudError, match="inputs drifted"): + session.validate_plan( + public["plan_id"], public["plan_sha256"], evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + + +def test_maintenance_journal_sequence_and_health_alone_never_prove_prune_completion(retention_env): + env = retention_env + records = _prepare_maintenance(env) + evidence, public, private = None, None, None + with _maintenance_context(env, records) as session: + evidence, public, private = _maintenance_plan(session) + with pytest.raises(CloudError, match="start with check_before"): + session.record_phase(private, "forget", "intent") + session.record_phase(private, "check_before", "intent") + with pytest.raises(CloudError, match="phase order"): + session.record_phase(private, "forget", "intent") + before = { + "repository_health": "healthy", + **evidence, + } + session.record_phase(private, "check_before", "completed", evidence=before) + session.record_phase(private, "forget", "intent") + with pytest.raises(CloudError, match="mutation already started"): + session.validate_plan( + public["plan_id"], public["plan_sha256"], evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + with pytest.raises(CloudError, match="earlier central maintenance attempt"): + session.plan( + evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + + result = session.finish_reconciliation( + private, + { + "schema_version": 1, + "repository_health": "healthy", + **evidence, + }, + ) + assert result["status"] == "partial_abandoned" + assert result["outcome"] == "abandoned" + assert result["remaining_target_snapshot_ids"] == ["b" * 64] + reconciled_journal = json.loads( + (env["state"] / "retention-operations" / f"{public['plan_id']}.json").read_text() + ) + assert reconciled_journal["phases"]["prune"] is None + assert reconciled_journal["status"] == "partial_abandoned" + next_plan = session.plan( + evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + assert next_plan["plan_id"] != public["plan_id"] + with pytest.raises(CloudError, match="mutation already started"): + session.validate_plan( + public["plan_id"], public["plan_sha256"], evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + + +def test_maintenance_fixed_phase_sequence_can_reconcile_complete(retention_env): + env = retention_env + records = _prepare_maintenance(env) + with _maintenance_context(env, records) as session: + _evidence, _public, private = _maintenance_plan(session) + target = ["b" * 64] + after = _restic_evidence(["a" * 64, "c" * 64]) + + session.record_phase(private, "check_before", "intent") + session.record_phase( + private, + "check_before", + "completed", + evidence={"repository_health": "healthy", **_evidence}, + ) + session.record_phase(private, "forget", "intent") + session.record_phase( + private, "forget", "completed", evidence={"forgotten_snapshot_ids": target} + ) + session.record_phase(private, "verify_snapshots", "intent") + session.record_phase(private, "verify_snapshots", "completed", evidence=after) + session.record_phase(private, "prune", "intent") + session.record_phase( + private, + "prune", + "completed", + evidence={"result": "completed", "evidence_sha256": "e" * 64}, + ) + session.record_phase(private, "check_after", "intent") + session.record_phase( + private, + "check_after", + "completed", + evidence={"repository_health": "healthy", **after}, + ) + session.record_phase(private, "protected_versions", "intent") + protected_versions = session.verify_protected_versions() + session.record_phase( + private, + "protected_versions", + "completed", + evidence=protected_versions, + ) + outcome = session.finish_reconciliation( + private, + {"schema_version": 1, "repository_health": "healthy", **after}, + ) + + assert outcome["status"] == "complete" + assert outcome["outcome"] == "succeeded" + assert outcome["remaining_target_snapshot_ids"] == [] + with _maintenance_context(env, records) as session: + fresh = session.plan( + after, bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + assert fresh["plan_id"] != _public["plan_id"] + assert fresh["target_snapshot_ids"] == [] + with pytest.raises(CloudError, match="mutation already started"): + session.validate_plan( + _public["plan_id"], _public["plan_sha256"], _evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}, + ) + + +def test_maintenance_pre_mutation_abandonment_allows_fresh_plan_not_replay(retention_env): + env = retention_env + records = _prepare_maintenance(env) + with _maintenance_context(env, records) as session: + evidence, original, private = _maintenance_plan(session) + session.record_phase(private, "check_before", "intent") + result = session.finish_reconciliation(private, + {"schema_version": 1, "repository_health": "healthy", **evidence}) + assert result["status"] == "abandoned" + fresh = session.plan(evidence, bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}) + assert fresh["plan_id"] != original["plan_id"] + with pytest.raises(CloudError, match="unresolved attempt"): + session.validate_plan(original["plan_id"], original["plan_sha256"], evidence, + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}) + + +def test_maintenance_terminal_resolution_requires_durable_healthy_evidence(retention_env): + env = retention_env + records = _prepare_maintenance(env) + with _maintenance_context(env, records) as session: + evidence, original, private = _maintenance_plan(session) + session.record_phase(private, "check_before", "intent") + session.finish_reconciliation(private, + {"schema_version": 1, "repository_health": "healthy", **evidence}) + journal_path = env["state"] / "retention-operations" / f"{original['plan_id']}.json" + journal = json.loads(journal_path.read_text()) + journal["reconciliation"]["repository_health"] = "unknown" + _write_json(journal_path, journal) + with pytest.raises(CloudError, match="existing central maintenance journal is invalid"): + session.plan(evidence, bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}) + + +def test_expired_maintenance_attempt_reconciles_read_only_against_original_protection(retention_env): + env = retention_env + records = _prepare_maintenance(env) + with _maintenance_context(env, records) as session: + evidence, public, private = _maintenance_plan(session) + session.record_phase(private, "check_before", "intent") + session.record_phase(private, "check_before", "completed", + evidence={"repository_health": "healthy", **evidence}) + session.record_phase(private, "forget", "intent") + FrozenDateTime.current += timedelta(days=60) + with pytest.raises(CloudError, match="expired"): + with _maintenance_context(env, records): + pytest.fail("expired authority must never reopen apply mode") + with env["adapter"].maintenance_session( + env["policy_path"], env["grant_path"], records, + quiescent_window_confirmed=True, repository_lease_id=env["repository_lease_id"], + reconcile_only=True, + ) as session: + reopened = session.reconciliation_plan(public["plan_id"], public["plan_sha256"], + bindings_sha256="f" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}) + assert reopened == private + with pytest.raises(CloudError, match="identity changed"): + session.reconciliation_plan(public["plan_id"], public["plan_sha256"], + bindings_sha256="e" * 64, + credential_identity={"key_id": "K0000000000000000000000001", "profile": "restricted"}) + outcome = session.finish_reconciliation(private, + {"schema_version": 1, "repository_health": "healthy", **evidence}) + assert outcome["status"] == "partial_abandoned" + assert outcome["protected_version_count"] == private["protected_version_count"] + + +def test_maintenance_reconciliation_requires_exact_protected_versions(retention_env): + env = retention_env + records = _prepare_maintenance(env) + with _maintenance_context(env, records) as session: + evidence, public, private = _maintenance_plan(session) + session.record_phase(private, "check_before", "intent") + session.record_phase( + private, + "check_before", + "completed", + evidence={"repository_health": "healthy", **evidence}, + ) + session.record_phase(private, "forget", "intent") + env["store"].versions = [ + row for row in env["store"].versions + if (row[0], row[1]) != ("repo/data/restore.bin", "historical-v1") + ] + with pytest.raises(CloudError, match="protected recovery checkpoint version"): + session.finish_reconciliation( + private, + { + "schema_version": 1, + "repository_health": "healthy", + **evidence, + }, + ) + + +def test_newly_accepted_unselected_checkpoint_protects_aged_version(retention_env): + env = retention_env + aged = _seed_and_age(env) + assert aged["candidate_count"] == 1 + second_dir, _manifest, _digest, _acceptance = _make_record( + env["root"] / "new-acceptance", "b" * 64, + [("repo/other/old.bin", "old-v1", b"old!")], + ) + env["policy"]["retention_counts"] = {"daily": 0, "weekly": 0, "monthly": 0} + _write_json(env["policy_path"], env["policy"]) + _refresh_grant_for_policy(env) + plan = _plan(env, [env["record_dir"], second_dir]) + assert plan["candidate_count"] == 0 + + +@pytest.mark.parametrize("damage", ["missing", "wrong_size"]) +def test_missing_or_changed_known_good_version_blocks_plan_and_apply(retention_env, damage): + env = retention_env + reviewed = _seed_and_age(env) + assert reviewed["candidate_count"] == 1 + previous_plans = set((env["state"] / "retention-plans").iterdir()) + if damage == "missing": + env["store"].versions = [ + row for row in env["store"].versions + if (row[0], row[1]) != ("repo/data/restore.bin", "historical-v1") + ] + else: + env["store"].versions = [ + (key, version, latest, 9 if version == "historical-v1" else size) + for key, version, latest, size in env["store"].versions + ] + with pytest.raises(CloudError, match="protected recovery checkpoint version"): + _plan(env) + with pytest.raises(CloudError, match="protected recovery checkpoint version"): + _apply(env, reviewed) + assert set((env["state"] / "retention-plans").iterdir()) == previous_plans + assert not (env["state"] / "retention-operations" / f"{reviewed['operation_id']}.json").exists() + assert env["store"].deleted == [] + + +@pytest.mark.parametrize("outcome", ["complete", "stopped", "cancelled"]) +def test_definite_apply_outcomes_release_repository_lease(retention_env, outcome): + from threading import Event + + env = retention_env + plan = _seed_and_age(env) + if outcome == "stopped": + original_hold = env["store"].get_object_legal_hold + checks = 0 + + def newly_held(**kwargs): + nonlocal checks + checks += 1 + if checks >= 2: + env["store"].legal_holds[("repo/other/old.bin", "old-v1")] = "ON" + return original_hold(**kwargs) + + env["store"].get_object_legal_hold = newly_held + cancellation = Event() + if outcome == "cancelled": + cancellation.set() + result = env["adapter"].apply_central_retention( + plan["operation_id"], plan["plan_sha256"], env["policy_path"], + env["grant_path"], [env["record_dir"]], + quiescent_window_confirmed=True, ledger=env["ledger"], + cancel_event=cancellation, mutation_client=env["store"], + ) + assert result["status"] == outcome + _assert_repository_reacquirable(env) + + +def test_busy_repository_blocks_scan_and_journal(retention_env, monkeypatch): + env = retention_env + plan = _seed_and_age(env) + holder = env["ledger"].begin(OperationPlan( + operation_id=uuid.uuid4().hex, repository_id=env["repository_lease_id"], + label="holder", argv=("hold",), destructive=True, + )) + try: + monkeypatch.setattr(cloud._CentralRetention, "_plan_inputs", lambda *_a, **_k: pytest.fail("scan while busy")) + with pytest.raises(RepositoryConflictError): + _apply(env, plan) + assert not (env["state"] / "retention-operations" / f"{plan['operation_id']}.json").exists() + assert env["store"].deleted == [] + finally: + holder.finish(OperationStatus.SUCCEEDED, 0) + holder.close() + + +def test_apply_and_reconcile_require_expected_service_repository_lease(retention_env): + env = retention_env + plan = _seed_and_age(env) + wrong = repository_key("different-service-repository") + with pytest.raises(CloudError, match="different repository lease"): + env["adapter"].apply_central_retention( + plan["operation_id"], plan["plan_sha256"], env["policy_path"], + env["grant_path"], [env["record_dir"]], quiescent_window_confirmed=True, + ledger=env["ledger"], mutation_client=env["store"], + expected_repository_lease_id=wrong, + ) + assert env["store"].deleted == [] + assert not (env["state"] / "retention-operations" / f"{plan['operation_id']}.json").exists() + assert _apply(env, plan)["status"] == "complete" + plan_path = env["state"] / "retention-plans" / f"{plan['operation_id']}.json" + with pytest.raises(CloudError, match="different repository lease"): + env["adapter"].reconcile_central_retention( + plan_path, ledger=env["ledger"], expected_repository_lease_id=wrong, + ) + + +def test_deadline_expiring_during_full_scan_fails_before_journal(retention_env, monkeypatch): + env = retention_env + env["policy"]["plan_max_age_seconds"] = 1 + _write_json(env["policy_path"], env["policy"]) + _refresh_grant_for_policy(env) + plan = _seed_and_age(env) + original = cloud._CentralRetention._inventory + + def slow_inventory(self, *args, **kwargs): + value = original(self, *args, **kwargs) + FrozenDateTime.current += timedelta(seconds=2) + return value + + monkeypatch.setattr(cloud._CentralRetention, "_inventory", slow_inventory) + with pytest.raises(CloudError, match="expired"): + _apply(env, plan) + assert env["store"].deleted == [] + assert not (env["state"] / "retention-operations" / f"{plan['operation_id']}.json").exists() + _assert_repository_reacquirable(env) + + +def test_deadline_between_candidates_stops_additional_deletes(retention_env): + env = retention_env + env["store"].versions.extend([ + ("repo/other2/old.bin", "old-v1", False, 4), + ("repo/other2/old.bin", "current-v2", True, 5), + ]) + env["policy"]["plan_max_age_seconds"] = 1 + _write_json(env["policy_path"], env["policy"]) + _refresh_grant_for_policy(env) + plan = _seed_and_age(env) + assert plan["candidate_count"] == 2 + original = env["store"].delete_object + + def slow_delete(**kwargs): + result = original(**kwargs) + FrozenDateTime.current += timedelta(seconds=2) + return result + + env["store"].delete_object = slow_delete + result = _apply(env, plan) + assert result["status"] == "stopped" + assert len(env["store"].deleted) == 1 + _assert_repository_reacquirable(env) + + +@pytest.mark.parametrize("boundary", ["lock_read", "intent_saved"]) +def test_cancellation_before_intent_or_delete_dispatches_nothing(retention_env, monkeypatch, boundary): + from threading import Event + + env = retention_env + plan = _seed_and_age(env) + cancellation = Event() + if boundary == "lock_read": + original_lock = cloud._CentralRetention._version_lock + reads = 0 + + def cancel_on_lock(self, *args, **kwargs): + nonlocal reads + result = original_lock(self, *args, **kwargs) + reads += 1 + if reads == 2: + cancellation.set() + return result + + monkeypatch.setattr(cloud._CentralRetention, "_version_lock", cancel_on_lock) + else: + original_save = cloud._CentralRetention._save_journal + + def cancel_after_intent(path, journal): + original_save(path, journal) + if journal["items"][0]["status"] == "intent": + cancellation.set() + + monkeypatch.setattr(cloud._CentralRetention, "_save_journal", staticmethod(cancel_after_intent)) + result = _apply(env, plan, cancel_event=cancellation) + assert result["status"] == "cancelled" + assert result["deleted_count"] == 0 + assert env["store"].deleted == [] + journal = json.loads((env["state"] / "retention-operations" / f"{plan['operation_id']}.json").read_text()) + assert journal["status"] == "cancelled" + assert journal["items"][0]["status"] == "pending" + _assert_repository_reacquirable(env) + + +def test_real_signatures_authenticate_acceptance_and_manifest(retention_env, monkeypatch): + if not shutil.which("gpg"): + pytest.skip("requires gpg") + env = retention_env + home = env["root"] / "gnupg" + home.mkdir(mode=0o700) + base = ["gpg", "--homedir", str(home), "--batch", "--pinentry-mode", "loopback", "--passphrase", ""] + subprocess.run([*base, "--quick-generate-key", "retention disposable test", "ed25519", "sign", "1d"], check=True, capture_output=True) + keys = subprocess.run([*base, "--with-colons", "--list-secret-keys"], check=True, capture_output=True, text=True).stdout + fingerprint = next(line.split(":")[9] for line in keys.splitlines() if line.startswith("fpr:")) + subprocess.run([*base, "--quick-generate-key", "retention independent verifier", "ed25519", "sign", "1d"], check=True, capture_output=True) + keys = subprocess.run([*base, "--with-colons", "--list-secret-keys"], check=True, capture_output=True, text=True).stdout + verifier = [line.split(":")[9] for line in keys.splitlines() if line.startswith("fpr:")][-1] + monkeypatch.setenv("GNUPGHOME", str(home)) + monkeypatch.setattr(__import__("bbackup.recovery", fromlist=["verify_checkpoint"]), "verify_checkpoint", real_verify_checkpoint) + monkeypatch.setattr(__import__("bbackup.recovery", fromlist=["verify_recovery_acceptance"]), "verify_recovery_acceptance", real_verify_acceptance) + acceptance = dict(env["acceptance"]) + acceptance["source_signer"] = fingerprint + acceptance["verifier"] = verifier + _write_json(env["record_dir"] / "acceptance.json", acceptance) + for filename, signature, signer in (("checkpoint.jsonl", "checkpoint.sig", fingerprint), + ("acceptance.json", "acceptance.sig", verifier)): + subprocess.run([*base, "--local-user", signer, "--output", str(env["record_dir"] / signature), + "--yes", "--detach-sign", str(env["record_dir"] / filename)], check=True, capture_output=True) + env["policy"]["trusted_verifier"] = verifier + _write_json(env["policy_path"], env["policy"]) + _refresh_grant_for_policy(env) + try: + assert real_verify_acceptance(env["record_dir"], trusted_verifier=verifier, + repository_id=REPOSITORY_ID, snapshot="a" * 64) + assert real_verify_checkpoint(env["manifest"], env["record_dir"] / "checkpoint.sig", fingerprint) + plan = _seed_and_age(env) + assert plan["candidate_count"] == 1 + assert _apply(env, plan)["status"] == "complete" + finally: + subprocess.run(["gpgconf", "--homedir", str(home), "--kill", "gpg-agent"], capture_output=True) diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index d47c4e1..972dfcb 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -172,7 +172,7 @@ async def exercise(): await app._supervise("stop") await pilot.pause() asyncio.run(exercise()) - assert calls == [("jobs run", {"name": "daily"})] + assert calls == [("jobs run", {"name": "daily", "quiet_administrative_window_confirmed": False})] def test_invalid_existing_setup_cannot_enroll_without_a_successful_review(tmp_path, monkeypatch): diff --git a/tests/test_database_capture.py b/tests/test_database_capture.py index 4115416..d36d7bc 100644 --- a/tests/test_database_capture.py +++ b/tests/test_database_capture.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import hashlib import threading import time from pathlib import Path @@ -98,6 +99,44 @@ def test_postgresql_custom_dump_streams_to_private_temporary_file( assert exported is not None and not exported.exists() +def test_covered_postgresql_manifest_binds_export_and_file_inputs(tmp_path, monkeypatch): + clients = tmp_path / "clients" + clients.mkdir() + _script(clients, "pg_dump", "printf 'PGDMP-reviewed-database'\n") + monkeypatch.setenv("PATH", str(clients)) + binding = _postgres_binding(tmp_path) + recovery = tmp_path / "recovery" + recovery.mkdir() + material = _private_file(recovery / "roles.sql", "reviewed recovery") + reviewed = {"source_id": "database", "connection": "production", + "database": "app_db", "root_id": "app_root", "cluster_id": "primary", + "baseline_digest": "a" * 64, + "recovery_artifacts": [{"path": str(material), + "sha256": hashlib.sha256(material.read_bytes()).hexdigest(), + "root_id": "recovery_root"}], + "recovery_file_roots": [{"root_id": "recovery_root", "path": str(recovery)}]} + checks = [] + with capture_sources((Source("files", "files", (str(recovery),)), _source("postgresql")), + tmp_path, database_bindings={"production": binding}, + coverage_postgresql={"database": reviewed}, + validate_coverage=lambda: checks.append("checked")) as paths: + assert str(recovery) in paths + manifest = json.loads(Path(paths[-1]).read_text()) + entry = manifest["exports"][0] + exported = Path(entry["path"]) + assert entry["source_database"] == "app_db" + assert entry["coverage"] == reviewed + assert entry["export_sha256"] == hashlib.sha256(exported.read_bytes()).hexdigest() + assert checks == ["checked"] + + with pytest.raises(CaptureError, match="mapping differs"): + with capture_sources((_source("postgresql"),), tmp_path, + database_bindings={"production": binding}, + coverage_postgresql={"database": {**reviewed, + "database": "wrong"}}): + pytest.fail("mismatched logical source became a capture") + + @pytest.mark.parametrize("output", ["", "not-a-custom-archive"]) def test_postgresql_rejects_successful_client_with_invalid_archive( tmp_path, monkeypatch, output diff --git a/tests/test_linux_coverage.py b/tests/test_linux_coverage.py new file mode 100644 index 0000000..ee9834a --- /dev/null +++ b/tests/test_linux_coverage.py @@ -0,0 +1,508 @@ +"""Local fixture checks for the private Linux coverage lifecycle.""" +import json +import hashlib +import os +import sqlite3 +from pathlib import Path + +import pytest + +from bbackup.enrollment import policy_lease, write_bytes +from bbackup.linux_coverage import (CoverageError, InventoryPaths, check_coverage, + discover_coverage, evaluate_recovery_evidence, + restic_literal_exclusion, review_coverage, + validate_scope_policy) + + +def _private_json(path, value): + path.write_text(json.dumps(value), encoding="utf-8") + path.chmod(0o600) + return path + + +def _recipe(tmp_path, name="fixture_recipe"): + procedure = tmp_path / (name + ".procedure") + procedure.write_text("reinstall from fixture package inventory\n") + procedure.chmod(0o600) + procedure_digest = hashlib.sha256(procedure.read_bytes()).hexdigest() + evidence = _private_json(tmp_path / (name + ".evidence.json"), { + "schema_version": 1, "recipe_id": name, "recipe_sha256": procedure_digest, + "environment": "isolated-local-fixture", + "checks": {"procedure_executed": True, "result_verified": True}, + }) + recipe = {"reason": "reinstallable", "consequence": "rebuild on restore", + "recipe_id": name, "recipe_sha256": procedure_digest} + reference = {"recipe_id": name, "recipe_sha256": procedure_digest, + "procedure_path": str(procedure), "evidence_path": str(evidence), + "evidence_sha256": hashlib.sha256(evidence.read_bytes()).hexdigest()} + return recipe, reference + + +@pytest.fixture +def coverage_fixture(tmp_path): + root = tmp_path / "source" + root.mkdir() + (root / ".credential").write_text("do not report this password") + (root / "database.sqlite").write_bytes(b"sqlite fixture") + (root / "alias").symlink_to(".credential") + (root / "original").write_text("hardlink") + (root / "linked").hardlink_to(root / "original") + try: + (root / "original").setxattr("user.fixture", b"value") + except (AttributeError, OSError): + pass + inventory_dir = tmp_path / "inventory" + inventory_dir.mkdir() + passwd = inventory_dir / "passwd" + passwd.write_text(f"fixture:x:1000:1000::{root}:/bin/sh\n") + group = inventory_dir / "group" + group.write_text("fixture:x:1000:\n") + mountinfo = inventory_dir / "mountinfo" + mountinfo.write_text("1 1 0:1 / / rw - ext4 /dev/fixture rw\n") + packages = inventory_dir / "packages" + packages.write_text("Package: fixture\n") + units = inventory_dir / "units" + units.mkdir() + inventory = InventoryPaths(passwd, group, mountinfo, packages, units) + policy = {"schema_version": 1, "approved_roots": ["home"], + "require_privileged": False, "filesystem_boundaries": "explicit", + "metadata_expectations": {name: True for name in + ("numeric_owner", "mode", "acl", "xattr", "links")}} + package_recipe, package_evidence = _recipe(tmp_path) + declaration = _private_json(tmp_path / "declaration.json", { + "schema_version": 1, + "roots": [{"id": "home", "path": str(root), "source_id": "files", "adapter": "files"}], + "exclusions": [], "mount_decisions": [], "identity_decisions": [], + "applications": [], "package_reconstruction": package_recipe, + "reconstruction_evidence": [package_evidence], + }) + state = tmp_path / "state" + policy_bytes, binding_bytes = b'{"policy":1}', b'{"binding":1}' + write_bytes(state, policy_bytes, binding_bytes) + with policy_lease(state, create=True): + pass + return {"root": root, "inventory": inventory, "policy": policy, + "declaration": declaration, "state": state, + "policy_bytes": policy_bytes, "binding_bytes": binding_bytes} + + +def _call(function, fixture, *args): + return function(fixture["declaration"], fixture["policy"], *args, + state=fixture["state"], policy_bytes=fixture["policy_bytes"], + binding_bytes=fixture["binding_bytes"], inventory=fixture["inventory"]) + + +def _review(fixture): + candidate = _call(discover_coverage, fixture) + assert candidate["coverage_status"] == "complete" + reviewed = _call(review_coverage, fixture, candidate["candidate_digest"]) + return candidate, reviewed + + +def _sqlite_root(coverage_fixture): + root = coverage_fixture["root"] + (root / "database.sqlite").unlink(missing_ok=True) + database_path = root / "database [review]*?.sqlite" + writer = sqlite3.connect(database_path) + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("CREATE TABLE committed (value TEXT)") + writer.execute("INSERT INTO committed VALUES ('first')") + writer.commit() + try: + os.setxattr(database_path, "user.coverage", b"original") + except (AttributeError, OSError): + writer.close() + pytest.skip("Fixture filesystem cannot store SQLite xattrs") + (coverage_fixture["inventory"].application_units / "fixture.service").write_text( + "[Service]\n") + declaration = json.loads(coverage_fixture["declaration"].read_text()) + recipe, evidence = _recipe(database_path.parent, "sqlite_rebuild") + declaration["roots"].append({"id": "database", "path": str(database_path), + "source_id": "database", "adapter": "sqlite"}) + declaration["applications"] = [{"id": "fixture", "unit": "fixture", + "source_id": "database", "state_roots": [str(database_path)], + "startup_recipe": recipe}] + declaration["reconstruction_evidence"].append(evidence) + _private_json(coverage_fixture["declaration"], declaration) + coverage_fixture["policy"] = {**coverage_fixture["policy"], + "approved_roots": ["home", "database"]} + return database_path, writer + + +def test_exact_review_capture_readiness_and_new_files(coverage_fixture): + candidate, reviewed = _review(coverage_fixture) + checked = _call(check_coverage, coverage_fixture) + assert checked.capture_ready + assert checked.baseline_digest == reviewed["baseline_digest"] + assert checked.approved_roots == (("home", coverage_fixture["root"], "files", "files"),) + assert "do not report" not in json.dumps(checked.public_result()) + assert str(coverage_fixture["root"]) not in json.dumps(candidate) + assert str(coverage_fixture["root"]) not in json.dumps(checked.public_result()) + (coverage_fixture["root"] / ".new-hidden-secret").write_text("new secret") + assert _call(check_coverage, coverage_fixture).capture_ready + + +def test_reviewed_sqlite_root_ignores_wal_churn_and_excludes_raw_files(coverage_fixture): + database_path, writer = _sqlite_root(coverage_fixture) + try: + _, reviewed = _review(coverage_fixture) + checked = _call(check_coverage, coverage_fixture) + assert checked.capture_ready + assert checked.baseline_digest == reviewed["baseline_digest"] + assert len(checked.approved_sqlite) == 1 + accepted = checked.approved_sqlite[0] + assert accepted["root_id"] == "database" + assert accepted["path"] == str(database_path) + assert accepted["identity"]["inode"] == database_path.stat().st_ino + assert accepted["identity"]["xattrs"] == [ + {"name": "user.coverage", "value_base64": "b3JpZ2luYWw="}] + assert accepted["recipe_digest"] == json.loads( + coverage_fixture["declaration"].read_text())["applications"][0][ + "startup_recipe"]["recipe_sha256"] + assert accepted["baseline_digest"] == reviewed["baseline_digest"] + assert set(checked.sqlite_capture_exclusions) == { + database_path, Path(str(database_path) + "-wal"), + Path(str(database_path) + "-shm"), Path(str(database_path) + "-journal")} + literal = restic_literal_exclusion(database_path) + assert "\\[" in literal and "\\]" in literal and "\\*" in literal and "\\?" in literal + + writer.execute("INSERT INTO committed VALUES ('after review')") + writer.commit() + assert Path(str(database_path) + "-wal").exists() + churned = _call(check_coverage, coverage_fixture) + assert churned.capture_ready + assert churned.candidate_digest == checked.candidate_digest + finally: + writer.close() + + +@pytest.mark.parametrize("change", ["inode", "mode"]) +def test_reviewed_sqlite_identity_and_metadata_drift(coverage_fixture, change): + database_path, writer = _sqlite_root(coverage_fixture) + writer.close() + _review(coverage_fixture) + if change == "inode": + replacement = coverage_fixture["root"].parent / "replacement.sqlite" + with sqlite3.connect(replacement) as database: + database.execute("CREATE TABLE replacement(value TEXT)") + os.replace(replacement, database_path) + else: + database_path.chmod(database_path.stat().st_mode & 0o777 ^ 0o100) + checked = _call(check_coverage, coverage_fixture) + assert checked.coverage_status == "drift" + assert "structural_inventory_changed" in checked.reasons + + +@pytest.mark.parametrize("failure", ["missing", "corrupt"]) +def test_missing_or_corrupt_sqlite_root_is_incomplete_before_review(coverage_fixture, failure): + database_path, writer = _sqlite_root(coverage_fixture) + writer.close() + if failure == "missing": + database_path.unlink() + else: + database_path.write_bytes(b"not a SQLite database") + candidate = _call(discover_coverage, coverage_fixture) + assert candidate["coverage_status"] == "incomplete" + expected = "sqlite_database_missing" if failure == "missing" else "sqlite_database_invalid" + assert expected in candidate["reasons"] + + +@pytest.mark.parametrize("unsafe", ["symlink", "directory", "hardlink"]) +def test_unsafe_existing_sqlite_sidecar_is_incomplete(coverage_fixture, unsafe): + database_path, writer = _sqlite_root(coverage_fixture) + writer.close() + sidecar = Path(str(database_path) + "-wal") + outside = coverage_fixture["root"].parent / "outside-sidecar" + outside.write_bytes(b"sidecar") + if unsafe == "symlink": + sidecar.symlink_to(outside) + elif unsafe == "directory": + sidecar.mkdir() + else: + sidecar.hardlink_to(outside) + candidate = _call(discover_coverage, coverage_fixture) + assert candidate["coverage_status"] == "incomplete" + assert "sqlite_sidecar_unsafe" in candidate["reasons"] + + +def test_sqlite_requires_exact_application_state_and_verified_recipe(coverage_fixture): + database_path, writer = _sqlite_root(coverage_fixture) + writer.close() + declaration = json.loads(coverage_fixture["declaration"].read_text()) + declaration["applications"][0]["state_roots"] = [str(database_path), str(coverage_fixture["root"])] + _private_json(coverage_fixture["declaration"], declaration) + candidate = _call(discover_coverage, coverage_fixture) + assert "sqlite_application_state_unaccounted" in candidate["reasons"] + + declaration["applications"][0]["state_roots"] = [str(database_path)] + declaration["reconstruction_evidence"] = [item for item in declaration["reconstruction_evidence"] + if item["recipe_id"] != declaration["applications"][0]["startup_recipe"]["recipe_id"]] + _private_json(coverage_fixture["declaration"], declaration) + candidate = _call(discover_coverage, coverage_fixture) + assert "reconstruction_evidence_missing" in candidate["reasons"] + + +def test_user_exclusion_cannot_cover_reviewed_sqlite_state(coverage_fixture, tmp_path): + database_path, writer = _sqlite_root(coverage_fixture) + writer.close() + declaration = json.loads(coverage_fixture["declaration"].read_text()) + recipe, evidence = _recipe(tmp_path, "exclude_sqlite") + declaration["exclusions"] = [{"root_id": "home", "path": str(database_path), + "recipe": recipe}] + declaration["reconstruction_evidence"].append(evidence) + _private_json(coverage_fixture["declaration"], declaration) + candidate = _call(discover_coverage, coverage_fixture) + assert "sqlite_state_excluded" in candidate["reasons"] + + +def test_sqlite_sidecar_cannot_be_an_explicit_files_root(coverage_fixture): + database_path, writer = _sqlite_root(coverage_fixture) + try: + wal_path = Path(str(database_path) + "-wal") + assert wal_path.is_file() + declaration = json.loads(coverage_fixture["declaration"].read_text()) + declaration["roots"].append({"id": "wal", "path": str(wal_path), + "source_id": "files", "adapter": "files"}) + _private_json(coverage_fixture["declaration"], declaration) + coverage_fixture["policy"] = {**coverage_fixture["policy"], + "approved_roots": ["home", "database", "wal"]} + candidate = _call(discover_coverage, coverage_fixture) + assert candidate["coverage_status"] == "incomplete" + assert "sqlite_artifact_explicitly_mapped" in candidate["reasons"] + finally: + writer.close() + + +@pytest.mark.parametrize("change", ["user", "mount", "app"]) +def test_structural_inventory_drift(coverage_fixture, change): + _review(coverage_fixture) + inventory = coverage_fixture["inventory"] + if change == "user": + inventory.passwd.write_text(inventory.passwd.read_text() + "new:x:1001:1001::/new:/bin/sh\n") + elif change == "mount": + inventory.mountinfo.write_text(inventory.mountinfo.read_text() + + f"2 1 0:2 / {coverage_fixture['root']} rw - ext4 /dev/other rw\n") + else: + (inventory.application_units / "new.service").write_text("[Service]\n") + checked = _call(check_coverage, coverage_fixture) + assert checked.coverage_status == "drift" + assert not checked.capture_ready + assert "structural_inventory_changed" in checked.reasons + + +def test_instantiated_systemd_services_remain_reviewable(coverage_fixture, tmp_path): + unit_stems = ["serial-getty@ttyS0", "worker@", r"worker@tty\x2dS0", "a" * 247] + inventory = coverage_fixture["inventory"].application_units + declaration = json.loads(coverage_fixture["declaration"].read_text()) + applications = [] + for index, stem in enumerate(unit_stems): + (inventory / f"{stem}.service").write_text("[Service]\n") + recipe, evidence = _recipe(tmp_path, f"startup_{index}") + declaration["reconstruction_evidence"].append(evidence) + applications.append({"id": f"unit_{index}", "unit": stem, + "source_id": "files", "state_roots": [str(coverage_fixture["root"])], + "startup_recipe": recipe}) + declaration["applications"] = applications + _private_json(coverage_fixture["declaration"], declaration) + + candidate, reviewed = _review(coverage_fixture) + checked = _call(check_coverage, coverage_fixture) + + assert candidate["counts"]["applications"] == len(unit_stems) + assert checked.capture_ready + assert checked.baseline_digest == reviewed["baseline_digest"] + + +@pytest.mark.parametrize("unit_stem", [ + "", "@ttyS0", "worker/ttyS0", "worker@ttyS0/extra", "../escape", "worker\nservice", + "worker@ttyS0@ttyS1", r"worker@tty\x2", r"worker@tty\xG0", + r"worker@tty\x00", "a" * 248, +]) +def test_invalid_systemd_service_stems_are_rejected(coverage_fixture, unit_stem): + declaration = json.loads(coverage_fixture["declaration"].read_text()) + recipe, evidence = _recipe(coverage_fixture["declaration"].parent, "invalid_unit") + declaration["applications"] = [{"id": "invalid_unit", "unit": unit_stem, + "source_id": "files", "state_roots": [str(coverage_fixture["root"])], + "startup_recipe": recipe}] + declaration["reconstruction_evidence"].append(evidence) + _private_json(coverage_fixture["declaration"], declaration) + + with pytest.raises(CoverageError, match="application unit"): + _call(discover_coverage, coverage_fixture) + + +def test_private_tmp_namespace_mount_ids_do_not_invalidate_reviewed_scope(coverage_fixture): + root = coverage_fixture["root"] + ancestor = root.parent + mountinfo = coverage_fixture["inventory"].mountinfo + def lines(start_id, *, private=False, bind=False): + text = (f"{start_id} {start_id} 0:1 / / rw - ext4 /dev/fixture rw\n" + f"{start_id + 1} {start_id} 0:2 / {ancestor} rw - ext4 /dev/data rw\n" + f"{start_id + 2} {start_id + 1} 0:3 / {root} rw - ext4 /dev/scope rw\n") + if private: + text += (f"{start_id + 3} {start_id + 1} 0:4 / {ancestor / 'systemd-private-tmp'} " + "rw - tmpfs tmpfs rw\n") + if bind: + text += (f"{start_id + 4} {start_id + 2} 0:5 / {root / 'bind'} " + "rw - ext4 /dev/bind rw\n") + return text + mountinfo.write_text(lines(10)) + declaration = json.loads(coverage_fixture["declaration"].read_text()) + declaration["mount_decisions"] = [{"mount_id": 12, "mountpoint": str(root), + "decision": "include"}] + _private_json(coverage_fixture["declaration"], declaration) + _review(coverage_fixture) + baseline_check = _call(check_coverage, coverage_fixture) + mountinfo.write_text(lines(110, private=True)) + namespace_check = _call(check_coverage, coverage_fixture) + assert namespace_check.capture_ready + assert namespace_check.candidate_digest == baseline_check.candidate_digest + mountinfo.write_text(lines(110, private=True).replace("/dev/scope", "/dev/replacement")) + assert _call(check_coverage, coverage_fixture).coverage_status == "drift" + (root / "bind").mkdir() + mountinfo.write_text(lines(110, private=True, bind=True)) + changed = _call(check_coverage, coverage_fixture) + assert changed.coverage_status == "drift" + assert "mount_boundary_undecided" in changed.reasons + + +def test_stale_candidate_and_baseline_tamper_fail_closed(coverage_fixture): + candidate = _call(discover_coverage, coverage_fixture) + coverage_fixture["inventory"].passwd.write_text("other:x:1000:1000::/missing:/bin/sh\n") + with pytest.raises(CoverageError, match="changed"): + _call(review_coverage, coverage_fixture, candidate["candidate_digest"]) + coverage_fixture["inventory"].passwd.write_text( + f"fixture:x:1000:1000::{coverage_fixture['root']}:/bin/sh\n") + _, reviewed = _review(coverage_fixture) + baseline = coverage_fixture["state"] / "coverage" / "baselines" / (reviewed["baseline_digest"] + ".json") + baseline.write_text("{}") + assert _call(check_coverage, coverage_fixture).coverage_status == "unenrolled" + + +def test_missing_root_and_changed_declaration_fail(coverage_fixture): + _review(coverage_fixture) + root = coverage_fixture["root"] + os.mkfifo(root / "unsupported") + assert _call(check_coverage, coverage_fixture).coverage_status == "incomplete" + (root / "unsupported").unlink() + declaration = json.loads(coverage_fixture["declaration"].read_text()) + declaration["roots"][0]["path"] = str(root / "elsewhere") + _private_json(coverage_fixture["declaration"], declaration) + assert _call(check_coverage, coverage_fixture).coverage_status == "drift" + + +def test_policy_rejects_incomplete_metadata_and_evidence_is_only_emulated(tmp_path): + with pytest.raises(CoverageError): + validate_scope_policy({"schema_version": 1, "approved_roots": ["x"]}) + evidence = _private_json(tmp_path / "evidence.json", { + "schema_version": 1, "environment": "isolated-local-fixture", + "baseline_digest": "a" * 64, "snapshot_id": "b" * 64, + "replacement_machine_digest": "c" * 64, "independent_escrow": True, + "checks": {name: True for name in + ("metadata", "secrets", "users", "reinstall", "app_data", "startup")}, + }) + result = evaluate_recovery_evidence(evidence, baseline_digest="a" * 64, + snapshot_id="b" * 64, source_machine_digest="d" * 64) + assert result["recovery_status"] == "emulated" + with pytest.raises(CoverageError, match="exact baseline"): + evaluate_recovery_evidence(evidence, baseline_digest="e" * 64, + snapshot_id="b" * 64, source_machine_digest="d" * 64) + + +def test_profiles_do_not_replace_each_others_active_review(coverage_fixture, tmp_path): + _review(coverage_fixture) + first = coverage_fixture["declaration"] + second = _private_json(tmp_path / "other-declaration.json", json.loads(first.read_text())) + coverage_fixture["declaration"] = second + _review(coverage_fixture) + assert _call(check_coverage, coverage_fixture).capture_ready + coverage_fixture["declaration"] = first + assert _call(check_coverage, coverage_fixture).capture_ready + + +def test_linux_account_shell_group_members_and_linked_home(coverage_fixture, tmp_path): + inventory = coverage_fixture["inventory"] + inventory.passwd.write_text(f"_apt:x:1000:1000::{coverage_fixture['root']}:/bin/sh\n") + inventory.group.write_text("_apt:x:1000:fixture\n") + _review(coverage_fixture) + inventory.group.write_text("_apt:x:1000:other\n") + assert _call(check_coverage, coverage_fixture).coverage_status == "drift" + inventory.group.write_text("_apt:x:1000:fixture\n") + inventory.passwd.write_text(f"_apt:x:1000:1000::{coverage_fixture['root']}:/bin/false\n") + assert _call(check_coverage, coverage_fixture).coverage_status == "drift" + outside = tmp_path / "outside" + outside.mkdir() + home_link = coverage_fixture["root"] / "home-link" + home_link.symlink_to(outside) + inventory.passwd.write_text(f"_apt:x:1000:1000::{home_link}:/bin/sh\n") + discovered = _call(discover_coverage, coverage_fixture) + assert discovered["coverage_status"] == "incomplete" + assert "user_home_unaccounted" in discovered["reasons"] + + +def test_nested_symlink_unit_and_unreadable_file_are_detected(coverage_fixture): + _review(coverage_fixture) + wants = coverage_fixture["inventory"].application_units / "multi-user.target.wants" + wants.mkdir() + (wants / "new.service").symlink_to("/usr/lib/systemd/system/new.service") + assert _call(check_coverage, coverage_fixture).coverage_status == "drift" + (wants / "new.service").unlink() + wants.rmdir() + if os.geteuid() == 0: + pytest.skip("Root can read mode-zero files") + secret = coverage_fixture["root"] / ".credential" + secret.chmod(0) + try: + discovered = _call(discover_coverage, coverage_fixture) + assert "unreadable_file" in discovered["reasons"] + finally: + secret.chmod(0o600) + + +def test_reconstruction_reference_must_be_verified(coverage_fixture): + declaration = json.loads(coverage_fixture["declaration"].read_text()) + declaration["reconstruction_evidence"] = [] + _private_json(coverage_fixture["declaration"], declaration) + discovered = _call(discover_coverage, coverage_fixture) + assert "reconstruction_evidence_missing" in discovered["reasons"] + declaration["reconstruction_evidence"] = [{"recipe_id": "fixture_recipe", + "recipe_sha256": declaration["package_reconstruction"]["recipe_sha256"], + "procedure_path": str(coverage_fixture["declaration"]), + "evidence_path": str(coverage_fixture["declaration"]), + "evidence_sha256": "a" * 64}] + _private_json(coverage_fixture["declaration"], declaration) + discovered = _call(discover_coverage, coverage_fixture) + assert "reconstruction_evidence_unverified" in discovered["reasons"] + + +def test_reviewed_exact_exclusion_is_carried_to_capture_gate(coverage_fixture, tmp_path): + declaration = json.loads(coverage_fixture["declaration"].read_text()) + recipe, evidence = _recipe(tmp_path, "credential_rebuild") + omitted = coverage_fixture["root"] / ".credential" + declaration["exclusions"] = [{"root_id": "home", "path": str(omitted), "recipe": recipe}] + declaration["reconstruction_evidence"].append(evidence) + _private_json(coverage_fixture["declaration"], declaration) + _review(coverage_fixture) + checked = _call(check_coverage, coverage_fixture) + assert checked.capture_ready + assert checked.approved_exclusions == (omitted,) + assert str(omitted) not in json.dumps(checked.public_result()) + omitted.unlink() + assert _call(check_coverage, coverage_fixture).coverage_status == "incomplete" + + +def test_application_state_link_outside_reviewed_roots_is_incomplete(coverage_fixture, tmp_path): + outside = tmp_path / "app-data" + outside.mkdir() + state_link = coverage_fixture["root"] / "app-data" + state_link.symlink_to(outside) + (coverage_fixture["inventory"].application_units / "fixture.service").write_text("[Service]\n") + declaration = json.loads(coverage_fixture["declaration"].read_text()) + recipe, evidence = _recipe(tmp_path, "startup") + declaration["applications"] = [{"id": "fixture", "unit": "fixture", + "source_id": "files", "state_roots": [str(state_link)], "startup_recipe": recipe}] + declaration["reconstruction_evidence"].append(evidence) + _private_json(coverage_fixture["declaration"], declaration) + discovered = _call(discover_coverage, coverage_fixture) + assert discovered["coverage_status"] == "incomplete" + assert "application_state_unaccounted" in discovered["reasons"] diff --git a/tests/test_models.py b/tests/test_models.py index feda1d3..928fb92 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -32,6 +32,57 @@ def test_defaults(config): assert parsed.jobs[0].schedule == "daily" +def test_coverage_opt_in_requires_portable_scope_and_exact_private_job_mapping(config, tmp_path): + from bbackup.models import HostBindings + scope = {"schema_version": 1, "approved_roots": ["data"], + "require_privileged": False, "filesystem_boundaries": "explicit", + "metadata_expectations": {name: True for name in + ("numeric_owner", "mode", "acl", "xattr", "links")}} + config["jobs"][0]["coverage_policy"] = scope + with pytest.raises(ConfigurationError, match="version 3"): + Configuration.parse(config) + config["schema_version"] = 3 + config["sources"] = [{"name": "files", "kind": "files"}] + parsed = Configuration.parse(config) + assert parsed.jobs[0].coverage_policy["approved_roots"] == ["data"] + raw = {"schema_version": 3, "state_dir": str(tmp_path / "state"), + "repositories": {name: {"repository": str(tmp_path / name), + "password_file": str(tmp_path / "password")} + for name in ("local", "cloud")}, + "source_bindings": {"files": {"paths": [str(tmp_path / "data")]}}, + "coverage_declarations": {"daily": str(tmp_path / "declaration.json")}} + bindings = HostBindings.parse(raw, parsed) + assert bindings.coverage_declarations["daily"] == tmp_path / "declaration.json" + del raw["coverage_declarations"] + with pytest.raises(ConfigurationError, match="opt-in jobs"): + HostBindings.parse(raw, parsed) + + +def test_remote_primary_requires_versioned_zero_cache_resource_budget(config): + config["repositories"][0]["role"] = "remote-primary" + with pytest.raises(ConfigurationError, match="resource budget"): + Configuration.parse(config) + config["jobs"][0]["resource_budget"] = { + "schema_version": 1, "staging_max_bytes": 1024**3, + "minimum_free_bytes": 64 * 1024**2, "cache_max_bytes": 0, + } + config["jobs"][0]["replicas"] = [] + parsed = Configuration.parse(config) + assert parsed.repositories[0].role == "remote-primary" + assert parsed.jobs[0].resource_budget.cache_max_bytes == 0 + config["jobs"][0]["replicas"] = ["cloud"] + with pytest.raises(ConfigurationError, match="replica copies"): + Configuration.parse(config) + config["jobs"][0]["replicas"] = [] + config["jobs"][0]["resource_budget"]["cache_max_bytes"] = 1 + with pytest.raises(ConfigurationError, match="cache"): + Configuration.parse(config) + config["jobs"][0]["resource_budget"]["cache_max_bytes"] = 0 + config["jobs"][0]["resource_budget"]["schema_version"] = 2 + with pytest.raises(ConfigurationError, match="version"): + Configuration.parse(config) + + @pytest.mark.parametrize( "text", ['{"a":1,"a":2}', '{"a":{"b":1,"b":2}}', '{"a":NaN}', '{"a":Infinity}'] ) diff --git a/tests/test_operations.py b/tests/test_operations.py index 46fdbb7..ef86335 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -86,6 +86,23 @@ def test_runner_cancels_child_when_event_is_set(tmp_path: Path) -> None: assert result.elapsed_seconds < 5 +def test_runner_tick_interrupts_silent_child(tmp_path: Path) -> None: + from bbackup.resource_limits import ResourceLimitError + calls = 0 + + def tick(): + nonlocal calls + calls += 1 + if calls >= 3: + raise ResourceLimitError("Observed free-space reserve fell below the configured threshold") + + with pytest.raises(ResourceLimitError, match="reserve"): + OperationRunner().run(OperationPlan.create(tmp_path / "repo", "silent", + (sys.executable, "-c", "import time; time.sleep(30)"), timeout_seconds=10), + on_tick=tick) + assert calls >= 3 + + def test_runner_streams_giant_line_in_bounded_chunks_and_caps_tails(tmp_path: Path) -> None: output = b"z" * 300_000 + b"\n" received: list[bytes] = [] @@ -109,6 +126,56 @@ def test_runner_streams_giant_line_in_bounded_chunks_and_caps_tails(tmp_path: Pa assert result.stderr_bytes == len(b"finished\n") +def test_runner_raw_stdout_hashes_binary_without_text_tail(tmp_path: Path) -> None: + payload = (b"private-sentinel-\xff\x00" * 1000) + observed = hashlib.sha256() + chunks = [] + code = "import os; os.write(1, b'private-sentinel-\\xff\\x00' * 1000)" + result = OperationRunner(read_chunk_bytes=127).run( + OperationPlan.create(tmp_path / "repo", "binary", (sys.executable, "-c", code)), + on_stdout_chunk=lambda chunk: (observed.update(chunk), chunks.append(len(chunk))), + ) + assert result.status is OperationStatus.SUCCEEDED + assert result.stdout_bytes == len(payload) == sum(chunks) + assert observed.hexdigest() == hashlib.sha256(payload).hexdigest() + assert result.stdout_tail == b"" + assert b"private-sentinel" not in repr(result).encode() + assert max(chunks) <= 127 + + +def test_runner_raw_stdout_bound_failure_reaps_child(tmp_path: Path) -> None: + observed = 0 + + def bounded(chunk): + nonlocal observed + observed += len(chunk) + if observed > 1024: + raise RuntimeError("bounded raw inspection refused excess bytes") + + code = "import os,time; os.write(1, b'x' * 100000); time.sleep(30)" + started = time.monotonic() + with pytest.raises(RuntimeError, match="bounded raw inspection"): + OperationRunner(read_chunk_bytes=512).run( + OperationPlan.create(tmp_path / "repo", "bounded", (sys.executable, "-c", code), + timeout_seconds=10), on_stdout_chunk=bounded) + assert time.monotonic() - started < 5 + + +def test_runner_raw_stdout_cancellation_keeps_tail_empty(tmp_path: Path) -> None: + cancel = threading.Event() + timer = threading.Timer(0.2, cancel.set) + timer.start() + try: + result = OperationRunner().run( + OperationPlan.create(tmp_path / "repo", "binary-cancel", + (sys.executable, "-c", "import os,time; os.write(1,b'private'); time.sleep(30)"), + timeout_seconds=10), on_stdout_chunk=lambda _chunk: None, cancel_event=cancel) + finally: + timer.cancel() + assert result.status is OperationStatus.CANCELLED + assert result.stdout_tail == b"" + + def test_runner_streams_file_backed_stdin_without_buffering_it(tmp_path: Path) -> None: payload = b"native-export" * 250_000 source = tmp_path / "large.sql" diff --git a/tests/test_postgresql_coverage.py b/tests/test_postgresql_coverage.py new file mode 100644 index 0000000..d89f362 --- /dev/null +++ b/tests/test_postgresql_coverage.py @@ -0,0 +1,664 @@ +"""Fixture tests for the bounded local PostgreSQL coverage owner.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from bbackup.enrollment import policy_lease, write_bytes +from bbackup.linux_coverage import (CoverageError, InventoryPaths, check_coverage, + discover_coverage, review_coverage) +from bbackup.models import DatabaseBinding, Source +from bbackup.operations import OperationStatus +from bbackup.postgresql_coverage import (PostgreSQLCoverageContext, + inspect_postgresql_coverage) + + +def _private(path: Path, content: bytes | str) -> Path: + if isinstance(content, str): + content = content.encode() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + path.chmod(0o600) + return path + + +def _digest(value) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=True).encode()).hexdigest() + + +def _nonconnectable_evidence(databases): + return [{"name": item["name"], "oid": item["oid"], + "metadata_sha256": _digest(item), + "checks": {"procedure_executed": True, "result_verified": True}} + for item in sorted(databases, key=lambda item: item["name"]) + if not item["allow_connections"]] + + +class _Runner: + """A fixture runner with the exact OperationRunner callback contract.""" + + def __init__(self, snapshot: dict, system_id: str = "7654321098765432109", + local_major: str = "16", *, oversized: bool = False): + self.snapshot = snapshot + self.system_id = system_id + self.local_major = local_major + self.oversized = oversized + self.calls = [] + self.per_database = {} + + def run(self, plan, *, on_stdout_chunk=None, cancel_event=None): + self.calls.append(plan.argv) + if plan.argv[0] == "psql": + database = next(arg for arg in plan.argv if arg.startswith("--dbname=")).rsplit("dbname=", 1)[1].strip("'") + snapshot = {**self.snapshot, **self.per_database.get(database, {}), + "current_database": database} + output = json.dumps(snapshot, sort_keys=True).encode() + b"\n" + elif plan.argv[0] == "pg_controldata" and plan.argv[1] == "--version": + output = f"pg_controldata (PostgreSQL) {self.local_major}.3\n".encode() + elif plan.argv[0] == "pg_controldata": + output = (f"pg_control version number: 1300\n" + f"Database system identifier: {self.system_id}\n" + "Database cluster state: in production\n").encode() + else: + raise AssertionError(f"unexpected native command: {plan.argv[0]}") + if self.oversized and plan.argv[0] == "psql": + output = b"x" * (1024 * 1024 + 1) + if cancel_event is not None and cancel_event.is_set(): + return SimpleNamespace(status=OperationStatus.CANCELLED, exit_code=None, + stdout_bytes=0) + if on_stdout_chunk is not None: + for offset in range(0, len(output), 32768): + on_stdout_chunk(output[offset:offset + 32768]) + return SimpleNamespace(status=OperationStatus.SUCCEEDED, exit_code=0, + stdout_bytes=len(output)) + + +def _recipe(directory: Path, name: str) -> tuple[dict, Path, str]: + procedure = _private(directory / f"{name}.procedure", "restore cluster from reviewed fixture\n") + digest = hashlib.sha256(procedure.read_bytes()).hexdigest() + return {"reason": "cluster reconstruction", "consequence": "rebuild cluster before application restore", + "recipe_id": name, "recipe_sha256": digest}, procedure, digest + + +def _snapshot(data_directory: Path) -> dict: + return { + "system_identifier": "7654321098765432109", + "server_version_num": 160003, + "data_directory": str(data_directory), + "config_file": str(data_directory / "postgresql.conf"), + "hba_file": str(data_directory / "pg_hba.conf"), + "ident_file": str(data_directory / "pg_ident.conf"), + "in_recovery": False, + "databases": [ + {"name": "app", "oid": 100, "allow_connections": True, "encoding": "UTF8", + "collate": "C", "ctype": "C", "locale_provider": "c", "owner": "postgres", + "acl": None, "connection_limit": -1, "is_template": False}, + {"name": "postgres", "oid": 5, "allow_connections": True, "encoding": "UTF8", + "collate": "C", "ctype": "C", "locale_provider": "c", "owner": "postgres", + "acl": None, "connection_limit": -1, "is_template": False}, + {"name": "template0", "oid": 4, "allow_connections": False, "encoding": "UTF8", + "collate": "C", "ctype": "C", "locale_provider": "c", "owner": "postgres", + "acl": None, "connection_limit": -1, "is_template": True}, + {"name": "template1", "oid": 1, "allow_connections": True, "encoding": "UTF8", + "collate": "C", "ctype": "C", "locale_provider": "c", "owner": "postgres", + "acl": None, "connection_limit": -1, "is_template": True}, + ], + "roles": [ + {"name": "postgres", "superuser": True, "inherit": True, "create_role": True, + "create_database": True, "can_login": True, "replication": True, + "bypass_rls": True, "connection_limit": -1, "valid_until": None, "password": None}, + {"name": "backup", "superuser": False, "inherit": True, "create_role": False, + "create_database": False, "can_login": True, "replication": False, + "bypass_rls": False, "connection_limit": -1, "valid_until": None, "password": None}, + ], + "memberships": [], + "extensions": [{"name": "plpgsql", "version": "1.0"}], + "languages": [], + "foreign_tables": 0, + "foreign_servers": 0, + "subscriptions": 0, + "replication_slots": 0, + "tablespaces": [{"name": "pg_default", "location": ""}, + {"name": "pg_global", "location": ""}], + "settings": [["max_connections", "100", "configuration file", "postmaster"]] + + [[name, value, "default", "postmaster"] for name, value in + (("ssl", "off"), ("archive_mode", "off"), + ("shared_preload_libraries", ""), ("session_preload_libraries", ""), + ("local_preload_libraries", ""), ("restore_command", ""), + ("archive_command", ""), ("primary_conninfo", ""), + ("primary_slot_name", ""))], + "role_settings": [{"database": "", "role": "backup", + "settings": ["application_name=private-test-setting"]}], + } + + +@pytest.fixture +def pg_fixture(tmp_path): + data_directory = tmp_path / "cluster" + data_directory.mkdir(mode=0o700) + data_directory.chmod(0o700) + (data_directory / "pg_wal").mkdir() + (data_directory / "pg_tblspc").mkdir() + _private(data_directory / "PG_VERSION", "16\n") + _private(data_directory / "postgresql.conf", "# fixture config\nmax_connections = 100\n") + _private(data_directory / "pg_hba.conf", "local all all trust\n") + _private(data_directory / "pg_ident.conf", "# fixture identity map\n") + service_file = _private(tmp_path / "pg_service.conf", "[local]\nuser=backup\n") + recovery_dir = tmp_path / "pg_recovery" + recovery_dir.mkdir(mode=0o700) + recovery_material = _private(recovery_dir / "roles.sql", "-- fixture roles restore\n") + material_sha = hashlib.sha256(recovery_material.read_bytes()).hexdigest() + recipe, procedure, procedure_sha = _recipe(recovery_dir, "pg_cluster_restore") + evidence_path = recovery_dir / "pg_cluster_restore.evidence.json" + # Initial evidence is deliberately stale so the fixture can calculate the + # exact inspection digests before writing a valid reviewed reconstruction record. + evidence = {"schema_version": 1, "recipe_id": "pg_cluster_restore", + "recipe_sha256": procedure_sha, "environment": "isolated-local-fixture", + "cluster_inventory_sha256": "0" * 64, "global_inventory_sha256": "0" * 64, + "role_inventory_sha256": "0" * 64, "configuration_sha256": "0" * 64, + "nonconnectable_databases": _nonconnectable_evidence(_snapshot(data_directory)["databases"]), + "recovery_material": [{"path": str(recovery_material), "sha256": material_sha}], + "checks": {"procedure_executed": True, "result_verified": True, + "quiet_administrative_window": True, "schema_changes_frozen": True}} + evidence_bytes = json.dumps(evidence, sort_keys=True).encode() + _private(evidence_path, evidence_bytes) + declaration = { + "schema_version": 2, + "roots": [{"id": "pg_recovery", "path": str(recovery_dir), + "source_id": "recovery_source", "adapter": "files"}], + "database_roots": [{"id": "app_db", "adapter": "postgresql", "source_id": "app_source", + "cluster_id": "primary", "connection": "pg_local", "database": "app"}], + "postgresql_clusters": [{"id": "primary", "connection": "pg_local", + "data_directory": str(data_directory), + "consistency_window": {"quiet_administrative_window": True, + "schema_changes_frozen": True}, + "databases": [ + {"name": "app", "oid": 100, "decision": "capture", "root_id": "app_db"}, + {"name": "postgres", "oid": 5, "decision": "reconstruct"}, + {"name": "template0", "oid": 4, "decision": "reconstruct"}, + {"name": "template1", "oid": 1, "decision": "reconstruct"}, + ], + "reconstruction": {"recipe": recipe, "procedure_path": str(procedure), + "evidence_path": str(evidence_path), + "evidence_sha256": hashlib.sha256(evidence_bytes).hexdigest(), + "recovery_material": [{"path": str(recovery_material), "sha256": material_sha}]}}], + "exclusions": [], "mount_decisions": [], "identity_decisions": [], + "applications": [], "package_reconstruction": None, "reconstruction_evidence": [], + } + declaration_path = _private(tmp_path / "coverage.json", json.dumps(declaration, sort_keys=True).encode()) + source = Source(name="app_source", kind="postgresql", database="app", connection="pg_local") + binding = DatabaseBinding("postgresql", service_file=service_file, service_name="local") + runner = _Runner(_snapshot(data_directory)) + context = PostgreSQLCoverageContext({"app_source": source}, {"pg_local": binding}, runner, + quiet_administrative_window=True) + inventory_dir = tmp_path / "inventory" + inventory_dir.mkdir() + passwd = _private(inventory_dir / "passwd", b"") + group = _private(inventory_dir / "group", b"") + mountinfo = _private(inventory_dir / "mountinfo", b"1 1 0:1 / / rw - ext4 /dev/fixture rw\n") + packages = _private(inventory_dir / "packages", b"Package: fixture\n") + units = inventory_dir / "units" + units.mkdir() + linux_inventory = InventoryPaths(passwd, group, mountinfo, packages, units) + package_procedure = _private(tmp_path / "package.procedure", "restore fixture packages\n") + package_sha = hashlib.sha256(package_procedure.read_bytes()).hexdigest() + package_evidence = {"schema_version": 1, "recipe_id": "package_restore", + "recipe_sha256": package_sha, "environment": "isolated-local-fixture", + "checks": {"procedure_executed": True, "result_verified": True}} + package_evidence_bytes = json.dumps(package_evidence, sort_keys=True).encode() + package_evidence_path = _private(tmp_path / "package.evidence.json", package_evidence_bytes) + declaration["package_reconstruction"] = {"reason": "fixture packages", + "consequence": "reinstall from inventory", "recipe_id": "package_restore", + "recipe_sha256": package_sha} + declaration["reconstruction_evidence"] = [{"recipe_id": "package_restore", + "recipe_sha256": package_sha, "procedure_path": str(package_procedure), + "evidence_path": str(package_evidence_path), + "evidence_sha256": hashlib.sha256(package_evidence_bytes).hexdigest()}] + declaration_path.write_text(json.dumps(declaration, sort_keys=True)) + declaration_path.chmod(0o600) + state = tmp_path / "state" + policy_bytes, binding_bytes = b'{"policy":1}', b'{"binding":1}' + write_bytes(state, policy_bytes, binding_bytes) + with policy_lease(state, create=True): + pass + policy = {"schema_version": 1, "approved_roots": ["app_db", "pg_recovery"], + "require_privileged": False, "filesystem_boundaries": "explicit", + "metadata_expectations": {key: True for key in + ("numeric_owner", "mode", "acl", "xattr", "links")}} + fixture = {"data_directory": data_directory, "declaration_path": declaration_path, + "declaration": declaration, "context": context, "runner": runner, + "policy": policy, "state": state, "policy_bytes": policy_bytes, + "binding_bytes": binding_bytes, "inventory": linux_inventory, + "evidence_path": evidence_path, "recovery_material": recovery_material} + _write_valid_pg_evidence(fixture) + return fixture + + +def _write_valid_pg_evidence(fixture): + declaration = fixture["declaration"] + pg = declaration["postgresql_clusters"][0] + inspected = inspect_postgresql_coverage(declaration["database_roots"], [pg], fixture["context"], + declaration["roots"], declaration["exclusions"]) + cluster = inspected["clusters"][0] + evidence = {"schema_version": 1, "recipe_id": pg["reconstruction"]["recipe"]["recipe_id"], + "recipe_sha256": pg["reconstruction"]["recipe"]["recipe_sha256"], + "environment": "isolated-local-fixture", + "cluster_inventory_sha256": cluster["cluster_inventory_sha256"], + "global_inventory_sha256": cluster["global_inventory_sha256"], + "role_inventory_sha256": cluster["role_inventory_sha256"], + "configuration_sha256": cluster["configuration_sha256"], + "nonconnectable_databases": _nonconnectable_evidence(cluster["databases"]), + "recovery_material": [{"path": str(fixture["recovery_material"]), + "sha256": hashlib.sha256(fixture["recovery_material"].read_bytes()).hexdigest()}], + "checks": {"procedure_executed": True, "result_verified": True, + "quiet_administrative_window": True, "schema_changes_frozen": True}} + evidence_bytes = json.dumps(evidence, sort_keys=True).encode() + _private(fixture["evidence_path"], evidence_bytes) + pg["reconstruction"]["evidence_sha256"] = hashlib.sha256(evidence_bytes).hexdigest() + fixture["declaration_path"].write_text(json.dumps(declaration, sort_keys=True)) + fixture["declaration_path"].chmod(0o600) + + +def _call(function, fixture, *args): + return function(fixture["declaration_path"], fixture["policy"], *args, + state=fixture["state"], policy_bytes=fixture["policy_bytes"], + binding_bytes=fixture["binding_bytes"], inventory=fixture["inventory"], + postgresql_context=fixture["context"]) + + +def _review(fixture): + candidate = _call(discover_coverage, fixture) + reviewed = _call(review_coverage, fixture, candidate["candidate_digest"]) + return candidate, reviewed + + +def _add_captured_database(fixture, name="analytics"): + snapshot = fixture["runner"].snapshot + snapshot["databases"].append({"name": name, "oid": 201, + "allow_connections": True, "encoding": "UTF8", "collate": "C", "ctype": "C", + "locale_provider": "c", "owner": "postgres", "acl": None, + "connection_limit": -1, "is_template": False}) + fixture["declaration"]["database_roots"].append({"id": "analytics_db", + "adapter": "postgresql", "source_id": "analytics_source", "cluster_id": "primary", + "connection": "pg_local", "database": name}) + fixture["declaration"]["postgresql_clusters"][0]["databases"].append( + {"name": name, "oid": 201, "decision": "capture", "root_id": "analytics_db"}) + fixture["policy"]["approved_roots"].append("analytics_db") + sources = {**fixture["context"].sources, "analytics_source": Source( + "analytics_source", "postgresql", database=name, connection="pg_local")} + fixture["context"] = PostgreSQLCoverageContext(sources, + fixture["context"].database_bindings, fixture["runner"], + quiet_administrative_window=True) + fixture["declaration_path"].write_text(json.dumps(fixture["declaration"], sort_keys=True)) + fixture["declaration_path"].chmod(0o600) + _write_valid_pg_evidence(fixture) + + +def test_schema2_reviews_logical_source_and_excludes_accounted_pgdata(pg_fixture): + candidate, reviewed = _review(pg_fixture) + checked = _call(check_coverage, pg_fixture) + assert candidate["coverage_status"] == "complete" + assert checked.capture_ready + assert reviewed["baseline_digest"] == checked.baseline_digest + assert checked.approved_roots[0][0] == "pg_recovery" + assert checked.approved_postgresql[0]["database"] == "app" + assert checked.approved_postgresql[0]["database_oid"] == 100 + assert checked.postgresql_capture_exclusions == (pg_fixture["data_directory"],) + assert checked.approved_exclusions == (pg_fixture["data_directory"],) + serialized = json.dumps(candidate, sort_keys=True) + assert "private-test-setting" not in serialized + assert "password" not in serialized.lower() + + +def test_selector_mismatch_and_unaccounted_database_fail_closed(pg_fixture): + source = pg_fixture["context"].sources["app_source"] + pg_fixture["context"] = PostgreSQLCoverageContext( + {"app_source": Source("app_source", "postgresql", database="other", connection="pg_local")}, + pg_fixture["context"].database_bindings, pg_fixture["runner"], + quiet_administrative_window=True) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_database_selector_mismatch" in candidate["reasons"] + pg_fixture["context"] = PostgreSQLCoverageContext({"app_source": source}, + pg_fixture["context"].database_bindings, pg_fixture["runner"], + quiet_administrative_window=True) + pg_fixture["runner"].snapshot["databases"].append( + {"name": "newdb", "oid": 200, "allow_connections": True, "encoding": "UTF8", + "collate": "C", "ctype": "C", "locale_provider": "c", "owner": "postgres", + "acl": None, "connection_limit": -1, "is_template": False}) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_database_accounting_changed" in candidate["reasons"] + + +def test_sql_system_identifier_must_match_local_control_file(pg_fixture): + pg_fixture["runner"].system_id = "7654321098765432190" + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_sql_local_identity_mismatch" in candidate["reasons"] + assert candidate["coverage_status"] == "incomplete" + + +@pytest.mark.parametrize("change,reason", [ + ("remote", "postgresql_remote_endpoint_unsupported"), + ("translated", "postgresql_data_directory_namespace_mismatch"), +]) +def test_remote_connection_and_path_translation_are_rejected(pg_fixture, change, reason): + if change == "remote": + service_file = pg_fixture["context"].database_bindings["pg_local"].service_file + _private(service_file, "[local]\nhost=db.example.invalid\nuser=backup\n") + else: + pg_fixture["runner"].snapshot["data_directory"] = str( + pg_fixture["data_directory"].parent / "translated-cluster") + pg_fixture["runner"].snapshot["config_file"] = str(pg_fixture["data_directory"] / "postgresql.conf") + pg_fixture["runner"].snapshot["hba_file"] = str(pg_fixture["data_directory"] / "pg_hba.conf") + pg_fixture["runner"].snapshot["ident_file"] = str(pg_fixture["data_directory"] / "pg_ident.conf") + candidate = _call(discover_coverage, pg_fixture) + assert reason in candidate["reasons"] + + +def test_changed_connection_material_and_identity_drift(pg_fixture): + _, reviewed = _review(pg_fixture) + service_file = pg_fixture["context"].database_bindings["pg_local"].service_file + _private(service_file, "[local]\nuser=another-local-role\n") + checked = _call(check_coverage, pg_fixture) + assert checked.coverage_status == "drift" + assert checked.baseline_digest == reviewed["baseline_digest"] + + +def test_reviewed_data_directory_inode_owner_mode_are_baselined(pg_fixture): + _review(pg_fixture) + pg_fixture["data_directory"].chmod(0o750) + checked = _call(check_coverage, pg_fixture) + assert checked.coverage_status == "drift" + + +@pytest.mark.parametrize("mutation,reason", [ + (lambda snapshot: snapshot["roles"].append({"name": "new_role", "superuser": False, + "inherit": True, "create_role": False, "create_database": False, "can_login": True, + "replication": False, "bypass_rls": False, "connection_limit": -1, "valid_until": None, + "password": None}), + "postgresql_reconstruction_evidence_stale"), + (lambda snapshot: snapshot["settings"].append(["work_mem", "8MB", "configuration file", "user"]), + "postgresql_reconstruction_evidence_stale"), + (lambda snapshot: snapshot.update({"extensions": [{"name": "fixture_ext", "version": "1"}]}), + "postgresql_extensions_unsupported"), + (lambda snapshot: snapshot.update({"in_recovery": True}), "postgresql_standby_unsupported"), +]) +def test_global_config_and_unsupported_state_are_detected(pg_fixture, mutation, reason): + mutation(pg_fixture["runner"].snapshot) + candidate = _call(discover_coverage, pg_fixture) + assert reason in candidate["reasons"] + + +@pytest.mark.parametrize("mutation", [ + lambda snapshot: snapshot["roles"].append({"name": "new_role", "superuser": False, + "inherit": True, "create_role": False, "create_database": False, "can_login": True, + "replication": False, "bypass_rls": False, "connection_limit": -1, "valid_until": None, + "password": None}), + lambda snapshot: snapshot["settings"].append(["work_mem", "8MB", "configuration file", "user"]), + lambda snapshot: snapshot["databases"][0].update({"oid": 101}), +]) +def test_reviewed_role_config_and_database_identity_changes_are_drift(pg_fixture, mutation): + _review(pg_fixture) + mutation(pg_fixture["runner"].snapshot) + checked = _call(check_coverage, pg_fixture) + assert checked.coverage_status == "drift" + + +def test_changed_recovery_material_invalidates_recipe_evidence(pg_fixture): + _review(pg_fixture) + _private(pg_fixture["recovery_material"], "changed material\n") + checked = _call(check_coverage, pg_fixture) + assert checked.coverage_status == "drift" + assert "postgresql_recovery_material_changed" in checked.reasons + + +def test_malformed_and_oversized_runner_output_fail_closed(pg_fixture): + pg_fixture["runner"].snapshot = {"unexpected": "shape"} + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_sql_inventory_malformed" in candidate["reasons"] + pg_fixture["runner"].oversized = True + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_inspection_output_limit" in candidate["reasons"] + + +def test_external_wal_and_configuration_are_rejected(pg_fixture, tmp_path): + wal = pg_fixture["data_directory"] / "pg_wal" + saved_wal = pg_fixture["data_directory"] / "pg_wal.saved" + wal.rename(saved_wal) + external_wal = tmp_path / "external-wal" + external_wal.mkdir() + wal.symlink_to(external_wal, target_is_directory=True) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_external_wal_unsupported" in candidate["reasons"] + + wal.unlink() + saved_wal.rename(wal) + external_config = _private(tmp_path / "external.conf", "max_connections = 100\n") + pg_fixture["runner"].snapshot["config_file"] = str(external_config) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_external_configuration" in candidate["reasons"] + + +def test_nested_mount_under_pgdata_is_rejected(pg_fixture): + nested = pg_fixture["data_directory"] / "pg_wal" + pg_fixture["inventory"].mountinfo.write_text( + f"1 1 0:1 / / rw - ext4 /dev/fixture rw\n" + f"2 1 0:2 / {nested} rw - tmpfs fixture rw\n") + pg_fixture["inventory"].mountinfo.chmod(0o600) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_external_storage_mount_unsupported" in candidate["reasons"] + + +def test_schema2_requires_all_capture_roots_and_recovery_recipe(pg_fixture): + declaration = pg_fixture["declaration"] + declaration["postgresql_clusters"][0]["databases"].pop() + pg_fixture["declaration_path"].write_text(json.dumps(declaration)) + pg_fixture["declaration_path"].chmod(0o600) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_database_accounting_changed" in candidate["reasons"] + del declaration["postgresql_clusters"][0]["reconstruction"]["recipe"] + pg_fixture["declaration_path"].write_text(json.dumps(declaration)) + pg_fixture["declaration_path"].chmod(0o600) + with pytest.raises(CoverageError): + _call(discover_coverage, pg_fixture) + + +def test_pgdata_conflicting_raw_root_is_rejected(pg_fixture, tmp_path): + declaration = pg_fixture["declaration"] + raw_path = pg_fixture["data_directory"] / "global" + raw_path.mkdir() + declaration["roots"].append({"id": "raw", "path": str(raw_path), + "source_id": "raw_source", "adapter": "files"}) + declaration["postgresql_clusters"][0]["data_directory"] = str(pg_fixture["data_directory"]) + pg_fixture["policy"]["approved_roots"] = ["raw", "app_db", "pg_recovery"] + pg_fixture["declaration_path"].write_text(json.dumps(declaration)) + pg_fixture["declaration_path"].chmod(0o600) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_raw_storage_root_conflict" in candidate["reasons"] + record = json.loads((pg_fixture["state"] / "coverage" / "candidates" / + f"{candidate['candidate_digest']}.json").read_text()) + assert record["postgresql_capture_exclusions"] == [] + + +def test_enclosing_files_root_receives_literal_pgdata_exclusion(pg_fixture, tmp_path): + enclosing = tmp_path / "enclosing" + enclosing.mkdir() + new_data_directory = enclosing / "cluster" + pg_fixture["data_directory"].rename(new_data_directory) + pg_fixture["data_directory"] = new_data_directory + declaration = pg_fixture["declaration"] + cluster = declaration["postgresql_clusters"][0] + cluster["data_directory"] = str(new_data_directory) + pg_fixture["runner"].snapshot["data_directory"] = str(new_data_directory) + pg_fixture["runner"].snapshot["config_file"] = str(new_data_directory / "postgresql.conf") + pg_fixture["runner"].snapshot["hba_file"] = str(new_data_directory / "pg_hba.conf") + pg_fixture["runner"].snapshot["ident_file"] = str(new_data_directory / "pg_ident.conf") + declaration["roots"].append({"id": "outer", "path": str(enclosing), + "source_id": "file_source", "adapter": "files"}) + pg_fixture["policy"]["approved_roots"] = ["outer", "app_db", "pg_recovery"] + pg_fixture["declaration_path"].write_text(json.dumps(declaration)) + pg_fixture["declaration_path"].chmod(0o600) + _write_valid_pg_evidence(pg_fixture) + candidate, _ = _review(pg_fixture) + checked = _call(check_coverage, pg_fixture) + assert candidate["coverage_status"] == "complete" + assert checked.capture_ready + assert checked.approved_exclusions == (new_data_directory,) + + +def test_postgresql_inspection_uses_bounded_native_runner(pg_fixture): + pg_fixture["runner"].calls.clear() + result = inspect_postgresql_coverage(pg_fixture["declaration"]["database_roots"], + pg_fixture["declaration"]["postgresql_clusters"], pg_fixture["context"], + pg_fixture["declaration"]["roots"], pg_fixture["declaration"]["exclusions"]) + assert not result["reasons"] + assert any(call[0] == "psql" for call in pg_fixture["runner"].calls) + assert sum(call[0] == "pg_controldata" for call in pg_fixture["runner"].calls) == 2 + query = next(arg for call in pg_fixture["runner"].calls if call[0] == "psql" + for arg in call if arg.startswith("--command=")) + assert "(pg_control_system()).system_identifier" in query + assert "d.oid::bigint" in query + assert "member_name.oid=m.member" in query + assert "m.memberid" not in query + + +def test_every_captured_database_is_inspected_and_second_only_extension_rejected(pg_fixture): + _add_captured_database(pg_fixture) + candidate, _ = _review(pg_fixture) + assert candidate["coverage_status"] == "complete" + assert len([call for call in pg_fixture["runner"].calls if call[0] == "psql"]) >= 2 + pg_fixture["runner"].per_database["analytics"] = { + "extensions": [{"name": "plpgsql", "version": "1.0"}, + {"name": "foreign_ext", "version": "1"}]} + checked = _call(check_coverage, pg_fixture) + assert "postgresql_extensions_unsupported" in checked.reasons + assert checked.coverage_status == "drift" + + +@pytest.mark.parametrize("database,override,reason", [ + ("postgres", {"extensions": [{"name": "plpgsql", "version": "1.0"}, + {"name": "foreign_ext", "version": "1"}]}, + "postgresql_extensions_unsupported"), + ("template1", {"foreign_servers": 1}, "postgresql_foreign_servers_unsupported"), +]) +def test_connectable_reconstructed_database_is_inspected(pg_fixture, database, override, reason): + pg_fixture["runner"].per_database[database] = override + candidate = _call(discover_coverage, pg_fixture) + assert reason in candidate["reasons"] + inspected = {arg.rsplit("dbname=", 1)[1].strip("'") + for call in pg_fixture["runner"].calls if call[0] == "psql" + for arg in call if arg.startswith("--dbname=")} + assert inspected >= {"app", "postgres", "template1"} + assert "template0" not in inspected + + +def test_nonconnectable_reconstruction_requires_database_bound_evidence(pg_fixture): + evidence = json.loads(pg_fixture["evidence_path"].read_text()) + evidence["nonconnectable_databases"] = [] + content = json.dumps(evidence, sort_keys=True).encode() + _private(pg_fixture["evidence_path"], content) + pg_fixture["declaration"]["postgresql_clusters"][0]["reconstruction"][ + "evidence_sha256"] = hashlib.sha256(content).hexdigest() + pg_fixture["declaration_path"].write_text(json.dumps(pg_fixture["declaration"])) + pg_fixture["declaration_path"].chmod(0o600) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_reconstruction_evidence_stale" in candidate["reasons"] + + +@pytest.mark.parametrize("capture_first", [True, False]) +def test_template_decision_order_does_not_change_connectability(pg_fixture, capture_first): + decisions = pg_fixture["declaration"]["postgresql_clusters"][0]["databases"] + decisions.sort(key=lambda item: item["decision"] == "capture", reverse=capture_first) + pg_fixture["declaration_path"].write_text(json.dumps(pg_fixture["declaration"])) + pg_fixture["declaration_path"].chmod(0o600) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_database_not_connectable" not in candidate["reasons"] + assert candidate["coverage_status"] == "complete" + + +@pytest.mark.parametrize("file,content,reason", [ + ("postgresql.conf", "include 'nested.conf'\n", "postgresql_external_configuration"), + ("pg_hba.conf", "include 'hba-extra.conf'\n", "postgresql_hba_dependencies_unsupported"), + ("postgresql.auto.conf", "ssl = 'on'\n", "postgresql_reconstruction_evidence_stale"), +]) +def test_configuration_dependencies_are_accounted(pg_fixture, file, content, reason): + path = pg_fixture["data_directory"] / file + _private(path, content) + candidate = _call(discover_coverage, pg_fixture) + assert reason in candidate["reasons"] + + +@pytest.mark.parametrize("archive_mode,complete", [("off", True), ("on", False)]) +def test_native_disabled_archive_command_requires_archiving_off(pg_fixture, archive_mode, complete): + settings = pg_fixture["runner"].snapshot["settings"] + next(item for item in settings if item[0] == "archive_command")[1] = "(disabled)" + next(item for item in settings if item[0] == "archive_mode")[1] = archive_mode + if complete: + _write_valid_pg_evidence(pg_fixture) + candidate = _call(discover_coverage, pg_fixture) + assert (candidate["coverage_status"] == "complete") is complete + assert ("postgresql_configuration_dependencies_unsupported" in candidate["reasons"]) is not complete + + +def test_recovery_artifact_must_be_inside_approved_file_root(pg_fixture): + pg_fixture["declaration"]["roots"] = [] + pg_fixture["policy"]["approved_roots"] = ["app_db"] + pg_fixture["declaration_path"].write_text(json.dumps(pg_fixture["declaration"])) + pg_fixture["declaration_path"].chmod(0o600) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_recovery_artifact_route_uncovered" in candidate["reasons"] + assert candidate["coverage_status"] == "incomplete" + + +def test_password_state_and_database_metadata_are_in_reviewed_inventory(pg_fixture): + _review(pg_fixture) + pg_fixture["runner"].snapshot["roles"][1]["password"] = "SCRAM-SHA-256$private" + checked = _call(check_coverage, pg_fixture) + assert "postgresql_reconstruction_evidence_stale" in checked.reasons + assert "SCRAM-SHA-256$private" not in json.dumps(checked.public_result()) + pg_fixture["runner"].snapshot["roles"][1]["password"] = None + pg_fixture["runner"].snapshot["databases"][0]["connection_limit"] = 7 + checked = _call(check_coverage, pg_fixture) + assert "postgresql_reconstruction_evidence_stale" in checked.reasons + + +def test_quiet_administrative_window_is_required_at_inspection_time(pg_fixture): + context = pg_fixture["context"] + pg_fixture["context"] = PostgreSQLCoverageContext(context.sources, + context.database_bindings, context.runner) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_quiet_administrative_window_required" in candidate["reasons"] + assert candidate["coverage_status"] == "incomplete" + + +def test_pg_hba_auth_file_reference_and_ssl_dependency_fail_closed(pg_fixture): + _private(pg_fixture["data_directory"] / "pg_hba.conf", "local all @roles trust\n") + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_hba_dependencies_unsupported" in candidate["reasons"] + _private(pg_fixture["data_directory"] / "pg_hba.conf", "local all all trust\n") + settings = pg_fixture["runner"].snapshot["settings"] + next(item for item in settings if item[0] == "ssl")[1] = "on" + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_configuration_dependencies_unsupported" in candidate["reasons"] + + +def test_custom_reported_hba_file_rejects_auth_reference(pg_fixture): + directory = pg_fixture["data_directory"] + custom = _private(directory / "auth.conf", "local all @external_roles trust\n") + pg_fixture["runner"].snapshot["hba_file"] = str(custom) + candidate = _call(discover_coverage, pg_fixture) + assert "postgresql_hba_dependencies_unsupported" in candidate["reasons"] + + +def test_approval_carries_exact_local_directory_identity(pg_fixture): + _review(pg_fixture) + checked = _call(check_coverage, pg_fixture) + identity = checked.approved_postgresql[0]["data_directory_identity"] + info = pg_fixture["data_directory"].stat() + assert identity == {"device": info.st_dev, "inode": info.st_ino, + "uid": info.st_uid, "gid": info.st_gid, + "mode": info.st_mode & 0o7777} diff --git a/tests/test_production_cli.py b/tests/test_production_cli.py index 98dd6ca..97f67ea 100644 --- a/tests/test_production_cli.py +++ b/tests/test_production_cli.py @@ -1,6 +1,8 @@ # ruff: noqa: F811 import json from dataclasses import asdict +from pathlib import Path +from types import SimpleNamespace from click.testing import CliRunner import pytest @@ -8,7 +10,7 @@ from bbackup.production_cli import _default_paths, cli from bbackup.configuration_workspace import ConfigurationWorkspace from bbackup.models import ConfigurationError -from bbackup.production_operations import execute_operation +from bbackup.production_operations import _service_operation, execute_operation from tests.test_service import service # noqa: F401 @@ -68,6 +70,325 @@ def test_schema_and_sanitized_listing(service): assert help_data["ui"]["activate"]["help"].startswith("Enable and start") +def test_manual_postgresql_window_flag_is_explicit_and_defaults_false(service, monkeypatch): + from bbackup.service import BackupService + observed = [] + + def run_job(self, name, *, cancel=None, quiet_administrative_window_confirmed=False): + observed.append((name, quiet_administrative_window_confirmed)) + return {"manual_window": quiet_administrative_window_confirmed} + + monkeypatch.setattr(BackupService, "run_job", run_job) + assert invoke(service, ["jobs", "run", "--name", "daily"]).exit_code == 0 + result = invoke(service, ["jobs", "run", "--name", "daily", + "--quiet-administrative-window-confirmed"]) + assert result.exit_code == 0 + assert observed == [("daily", False), ("daily", True)] + schema = json.loads(invoke(service, ["coverage", "discover", "--schema"]).output)["data"] + assert schema["properties"]["quiet_administrative_window_confirmed"]["type"] == "boolean" + + +@pytest.mark.parametrize("quiet_window", [False, True]) +def test_reconcile_cli_forwards_quiet_window(service, monkeypatch, quiet_window): + calls = [] + + def reconcile(self, repository, operation_id, outcome, *, + quiescent_window_confirmed=False, cancel=None): + calls.append((repository, operation_id, outcome.value, quiescent_window_confirmed)) + return {"replayed": False} + + monkeypatch.setattr(type(service), "reconcile_run", reconcile) + arguments = ["runs", "reconcile", "--repository", "local", "--operation-id", "a" * 32, + "--outcome", "failed", "--repository-reviewed"] + if quiet_window: + arguments.append("--quiescent-window-confirmed") + result = invoke(service, arguments) + assert result.exit_code == 0, result.output + assert calls == [("local", "a" * 32, "failed", quiet_window)] + + +def test_central_acceptance_dispatch_uses_independent_recovery_owner(monkeypatch): + from pathlib import Path + from bbackup import production_operations as module + + calls = [] + adapter = object() + monkeypatch.setattr(module, "S3CloudAdapter", lambda *args, **kwargs: adapter) + monkeypatch.setattr(module, "accept_recovery_checkpoint", lambda *args, **kwargs: calls.append( + (args, kwargs)) or {"protection_qualified": False}) + values = { + "kit": "/private/kit.gpg", "target": "/private/acceptance", "source_signer": "A" * 40, + "verifier": "B" * 40, "repository": "cloud", "repository_id": "a" * 64, + "snapshot": "b" * 64, "bucket": "backup-bucket", "endpoint": "https://s3.example.test", + } + result = execute_operation("recovery accept", values) + assert result == {"protection_qualified": False} + assert calls[0][0] == (adapter, Path(values["kit"]), Path(values["target"])) + assert calls[0][1]["trusted_verifier"] == values["verifier"] + assert calls[0][1]["source_repository"] == "cloud" + schema = execute_operation("skills", {"name": "recovery accept"})["recovery accept"]["input_schema"] + assert schema["additionalProperties"] is False + assert {"kit", "verifier", "source_signer", "repository_id", "snapshot"} <= set(schema["required"]) + + +def test_cloud_retention_dispatch_binds_target_and_existing_ledger(service, tmp_path, monkeypatch): + from dataclasses import replace + from threading import Event + from bbackup import production_operations as module + from bbackup.models import RepositoryBinding + from bbackup.operations import repository_key + + identity = "s3:https://retention.example.test/bucket/repo" + service.bindings.repositories["local"] = RepositoryBinding(identity, tmp_path / "password") + policy = tmp_path / "central-policy.json" + policy.write_text(json.dumps({"endpoint_url": "https://retention.example.test", "bucket": "bucket", "prefix": "repo/"})) + policy.chmod(0o600) + records = tmp_path / "records.json" + records.write_text(json.dumps([str(tmp_path / "accepted")])) + records.chmod(0o600) + calls = [] + class Adapter: + def __init__(self, *args, **kwargs): + calls.append(("adapter", args, kwargs)) + def initialize_central_retention_catalog(self, *args, **kwargs): + calls.append(("enroll", args, kwargs)) + return {"enrolled": True} + def plan_central_retention(self, *args, **kwargs): + calls.append(("plan", args, kwargs)) + return {"candidate_count": 0} + def apply_central_retention(self, *args, **kwargs): + calls.append(("apply", args, kwargs)) + return {"status": "applied"} + def reconcile_central_retention(self, *args, **kwargs): + calls.append(("reconcile", args, kwargs)) + return {"status": "reconciled_abandoned"} + monkeypatch.setattr(module, "S3CloudAdapter", Adapter) + common = {"repository": "local", "policy_file": str(policy)} + grant = {**common, "grant_file": str(tmp_path / "grant.json")} + window = {**grant, "records_file": str(records), "quiescent_window_confirmed": True} + cancel = Event() + def dispatch(command, values): + return module._service_operation(service, command, values, None, None, cancel) + assert dispatch("cloud-retention catalog-enroll", grant)["enrolled"] is True + assert dispatch("cloud-retention plan", window)["candidate_count"] == 0 + assert calls[-1][2]["repository_lease_id"] == repository_key(identity) + assert dispatch("cloud-retention apply", {**window, "plan_id": "a" * 32, "plan_sha256": "b" * 64})["status"] == "applied" + assert calls[-1][2]["ledger"] is service.ledger + assert calls[-1][2]["cancel_event"] is cancel + assert calls[-1][2]["expected_repository_lease_id"] == repository_key(identity) + dispatch("cloud-retention reconcile", {**common, "plan_file": str(tmp_path / "plan.json")}) + assert calls[-1][2] == {"ledger": service.ledger, "expected_repository_lease_id": repository_key(identity)} + service.bindings.repositories["local"] = replace(service.bindings.repositories["local"], repository=identity + "-other") + before = len(calls) + with pytest.raises(ConfigurationError, match="differs from enrolled"): + dispatch("cloud-retention apply", window) + assert len(calls) == before + + +def test_schema2_retention_freezes_one_pair_for_native_and_s3(service, tmp_path, monkeypatch): + from types import SimpleNamespace + from bbackup import production_operations as module + from bbackup.models import RepositoryBinding + import boto3 + + endpoint = "https://s3.us-west-004.backblazeb2.com" + service.bindings.repositories["local"] = RepositoryBinding( + f"s3:{endpoint}/backup-bucket/repo", tmp_path / "password", + ) + policy = tmp_path / "native-policy.json" + policy.write_text(json.dumps({ + "schema_version": 2, "endpoint_url": endpoint, "bucket": "backup-bucket", + "prefix": "repo/", "lock_metadata": {"provider": "b2-native", "bucket_id": "bucket-id-1"}, + })) + policy.chmod(0o600) + calls = [] + frozen = SimpleNamespace(access_key="key", secret_key="secret", token=None) + monkeypatch.setattr(boto3, "Session", lambda: SimpleNamespace( + get_credentials=lambda: SimpleNamespace(get_frozen_credentials=lambda: frozen))) + monkeypatch.setattr(boto3, "client", lambda *args, **kwargs: calls.append(("s3", kwargs)) or object()) + monkeypatch.setattr(module, "B2VersionLockReader", lambda *args, **kwargs: calls.append(("native", kwargs)) or object()) + class Adapter: + def __init__(self, *args, **kwargs): + calls.append(("adapter", kwargs)) + def initialize_central_retention_catalog(self, *_args): + return {"enrolled": True} + monkeypatch.setattr(module, "S3CloudAdapter", Adapter) + values = {"repository": "local", "policy_file": str(policy), "grant_file": str(tmp_path / "grant.json")} + assert module._cloud_retention_operation(service, "cloud-retention catalog-enroll", values, None)["enrolled"] + assert calls[0][0] == "native" and calls[1][0] == "s3" and calls[2][0] == "adapter" + assert calls[0][1]["access_key_id"] == calls[1][1]["aws_access_key_id"] == "key" + assert calls[0][1]["secret_access_key"] == calls[1][1]["aws_secret_access_key"] == "secret" + assert calls[2][1]["credential_pair"] == ("key", "secret") + assert calls[2][1]["region_name"] == "us-west-004" + frozen.token = "session-token" + calls.clear() + with pytest.raises(ConfigurationError, match="bind native B2"): + module._cloud_retention_operation(service, "cloud-retention catalog-enroll", values, None) + assert calls == [] + + +def test_schema3_maintenance_freezes_adapter_pair_without_native_metadata(service, tmp_path, monkeypatch): + from types import SimpleNamespace + from bbackup import production_operations as module + from bbackup.models import RepositoryBinding + import boto3 + + endpoint = "https://s3.us-west-004.backblazeb2.com" + service.bindings.repositories["local"] = RepositoryBinding( + f"s3:{endpoint}/backup-bucket/repo", tmp_path / "password") + policy = tmp_path / "policy.json" + policy.write_text(json.dumps({"schema_version": 3, "endpoint_url": endpoint, + "bucket": "backup-bucket", "prefix": "repo/"})) + policy.chmod(0o600) + calls = [] + frozen = SimpleNamespace(access_key="adapter-key", secret_key="adapter-secret", token=None) + monkeypatch.setattr(boto3, "Session", lambda: SimpleNamespace( + get_credentials=lambda: SimpleNamespace(get_frozen_credentials=lambda: frozen))) + monkeypatch.setattr(boto3, "client", lambda *_args, **kwargs: calls.append(("s3", kwargs)) or object()) + class Adapter: + def __init__(self, *_args, **kwargs): + calls.append(("adapter", kwargs)) + def initialize_central_retention_catalog(self, *_args, **kwargs): + return {"status": "enrolled"} + monkeypatch.setattr(module, "S3CloudAdapter", Adapter) + result = module._cloud_maintenance_operation(service, "cloud-maintenance catalog-enroll", + {"repository": "local", "policy_file": str(policy), "grant_file": str(tmp_path / "grant")}, None) + assert result["status"] == "enrolled" + assert calls[0][1]["aws_access_key_id"] == "adapter-key" + assert calls[1][1]["credential_pair"] == ("adapter-key", "adapter-secret") + assert calls[1][1]["native_lock_reader"] is None + + +def test_cloud_maintenance_cli_has_exact_private_input_schemas(): + for name in ("catalog-enroll", "records-enroll", "plan", "apply", "reconcile"): + definition = execute_operation("skills", {"name": "cloud-maintenance " + name}) + assert definition["cloud-maintenance " + name]["input_schema"]["additionalProperties"] is False + catalog = execute_operation("skills", {"name": "cloud-maintenance catalog-enroll"}) + assert set(catalog["cloud-maintenance catalog-enroll"]["input_schema"]["required"]) == { + "repository", "policy_file", "grant_file"} + apply = execute_operation("skills", {"name": "cloud-maintenance apply"}) + assert {"records_file", "maintenance_bindings_file", "plan_id", "plan_sha256"} <= set( + apply["cloud-maintenance apply"]["input_schema"]["required"]) + + +def test_cloud_retention_requires_enrollment_and_private_record_list(service, tmp_path, monkeypatch): + result = CliRunner().invoke(cli, ["cloud-retention", "apply", "--schema"]) + schema = json.loads(result.output)["data"] + assert {"repository", "policy_file", "grant_file", "records_file", "plan_sha256"} <= set(schema["required"]) + assert schema["additionalProperties"] is False + for key in ("catalog-enroll", "plan", "apply", "reconcile"): + definition = execute_operation("skills", {"name": "cloud-retention " + key}) + assert definition + with pytest.raises(ConfigurationError): + execute_operation("cloud-retention catalog-enroll", { + "repository": "local", "policy_file": str(tmp_path / "policy"), + "grant_file": str(tmp_path / "grant")}) + + +def test_coverage_cli_uses_enrolled_bytes_and_exclusive_review_owner(service, tmp_path, monkeypatch): + from dataclasses import replace + from bbackup import production_operations as module + scope = {"schema_version": 1, "approved_roots": ["home"], "require_privileged": False, + "filesystem_boundaries": "explicit", "metadata_expectations": { + key: True for key in ("numeric_owner", "mode", "acl", "xattr", "links")}} + service.config = replace(service.config, jobs=(replace( + service.config.jobs[0], coverage_policy=scope),)) + declaration = tmp_path / "private-coverage.json" + service.bindings = replace(service.bindings, coverage_declarations={"daily": declaration}) + calls = [] + def discover(path, policy, **kwargs): + assert path == declaration and policy == scope + assert b'"schema_version": 3' in kwargs["policy_bytes"] + assert str(declaration).encode() in kwargs["binding_bytes"] + calls.append("discover") + return {"coverage_status": "incomplete", "candidate_digest": "a" * 64} + def review(path, policy, candidate, **kwargs): + module.verify_bytes(kwargs["state"], kwargs["policy_bytes"], kwargs["binding_bytes"]) + assert candidate == "a" * 64 + calls.append("review") + return {"coverage_status": "complete", "recovery_status": "untested"} + monkeypatch.setattr(module, "discover_coverage", discover) + monkeypatch.setattr(module, "review_coverage", review) + result = invoke(service, ["coverage", "discover", "--name", "daily"]) + assert result.exit_code == 0, result.output + assert calls == ["discover"] + monkeypatch.setattr(module, "_operation_lease", lambda *_args: pytest.fail("nested shared lease")) + result = invoke(service, ["coverage", "review", "--name", "daily", "--candidate-digest", "a" * 64]) + assert result.exit_code == 0, result.output + assert calls == ["discover", "review"] + + +def test_postgresql_coverage_operations_forward_manual_window_and_cancel(monkeypatch): + from threading import Event + from bbackup import production_operations as module + + cancel = Event() + job = SimpleNamespace(name="daily", coverage_policy={"schema_version": 1}, + sources=("database",)) + pg_source = SimpleNamespace(name="database", kind="postgresql") + configuration = SimpleNamespace(jobs=(job,), sources=(pg_source,)) + resolved = SimpleNamespace(name="database", kind="postgresql") + bindings = SimpleNamespace(state_dir=Path("/private/state"), + coverage_declarations={"daily": Path("/private/declaration")}, + source_bindings={"database": SimpleNamespace(resolve=lambda _source: resolved)}, + database_bindings={"source_pg": object()}) + seen = [] + + def inspect(_path, _policy, **kwargs): + context = kwargs["postgresql_context"] + seen.append((context.quiet_administrative_window, context.cancel, + context.sources["database"] is resolved)) + return SimpleNamespace(capture_ready=True, baseline_digest="a" * 64, + public_result=lambda: {"coverage_status": "complete"}) + + monkeypatch.setattr(module, "discover_coverage", inspect) + monkeypatch.setattr(module, "review_coverage", lambda path, policy, candidate, **kw: + inspect(path, policy, **kw)) + monkeypatch.setattr(module, "check_coverage", inspect) + monkeypatch.setattr(module, "evaluate_recovery_evidence", lambda *_args, **_kwargs: {"ok": True}) + for command in ("coverage discover", "coverage review", "coverage check", + "coverage recovery-evidence"): + values = {"name": "daily", "quiet_administrative_window_confirmed": True, + "candidate_digest": "a" * 64, "snapshot": "b" * 64} + module._coverage_operation(configuration, bindings, b"policy", b"bindings", + command, values, cancel=cancel) + assert seen == [(True, cancel, True)] * 4 + + +def test_coverage_cli_sanitizes_errors_and_does_not_approve_an_ordinary_job(service, monkeypatch): + result = invoke(service, ["coverage", "check", "--name", "daily"]) + assert result.exit_code == 2 + assert json.loads(result.output)["errors"][0]["code"] == "invalid_input" + from bbackup import production_operations as module + from bbackup.linux_coverage import CoverageError + def failed(*args, **kwargs): + raise CoverageError("private paths must never escape the CLI") + monkeypatch.setattr(module, "execute_operation", failed) + # The CLI imports its owner function; exercise the sanitizing boundary. + monkeypatch.setattr("bbackup.production_cli.execute_operation", failed) + result = invoke(service, ["coverage", "discover", "--name", "daily"]) + assert result.exit_code == 3 and "private paths" not in result.output + assert json.loads(result.output)["errors"][0]["code"] == "operation_failed" + + +def test_resource_failure_keeps_structured_evidence_without_source_paths(service, monkeypatch): + from bbackup.service import BackupService, ResourceBudgetExceeded + + evidence = {"resource_budget_exceeded": True, "repository": "cloud", "replayed": False, + "local_complete": False, "cloud_complete": False} + + def fail(*args, **kwargs): + raise ResourceBudgetExceeded(evidence) + + monkeypatch.setattr(BackupService, "run_job", fail) + result = invoke(service, ["jobs", "run", "--name", "daily"]) + assert result.exit_code == 3 + payload = json.loads(result.output) + assert payload["success"] is False + assert payload["errors"][0]["code"] == "resource_budget_exceeded" + assert payload["data"] == evidence + + def test_cli_reports_enrollment_next_action(service): assert invoke(service, ["jobs", "list"]).exit_code == 0 config = service.state / "cli-config.json" @@ -167,6 +488,33 @@ def test_reconciliation_requires_explicit_review_and_does_not_run_command(servic assert json.loads(missing.output)["success"] is False +def test_retained_export_commands_have_closed_schema_and_route_exact_digest(service, monkeypatch): + listed = CliRunner().invoke(cli, ["runs", "retained-exports", "--schema"]) + assert listed.exit_code == 0 + listing_schema = json.loads(listed.output)["data"] + assert set(listing_schema["properties"]) == {"repository", "limit", "offset"} + disposal = CliRunner().invoke(cli, ["runs", "discard-exports", "--schema"]) + assert disposal.exit_code == 0 + disposal_schema = json.loads(disposal.output)["data"] + assert set(disposal_schema["required"]) == { + "repository", "operation_id", "artifact_record_sha256"} + assert disposal_schema["additionalProperties"] is False + calls = [] + monkeypatch.setattr(service, "retained_exports", lambda *args, **kwargs: calls.append( + ("list", args, kwargs)) or []) + monkeypatch.setattr(service, "discard_exports", lambda *args: calls.append( + ("discard", args)) or {"discarded_export_count": 1}) + assert _service_operation(service, "runs retained-exports", + {"repository": "primary", "limit": 5, "offset": 2}, + None, None, None) == [] + assert _service_operation(service, "runs discard-exports", { + "repository": "primary", "operation_id": "a" * 32, + "artifact_record_sha256": "b" * 64}, None, None, None) == { + "discarded_export_count": 1} + assert calls == [("list", ("primary",), {"limit": 5, "offset": 2}), + ("discard", ("primary", "a" * 32, "b" * 64))] + + def test_docker_command_schemas_are_closed_and_require_explicit_identity(service): expectations = { diff --git a/tests/test_recovery.py b/tests/test_recovery.py index 9f562aa..79b41b0 100644 --- a/tests/test_recovery.py +++ b/tests/test_recovery.py @@ -3,7 +3,8 @@ import pytest -from bbackup.recovery import RecoveryError, open_kit, seal_kit, verify_checkpoint +from bbackup.recovery import (RecoveryError, accept_recovery_checkpoint, open_kit, + seal_kit, verify_checkpoint, verify_recovery_acceptance) @pytest.fixture @@ -21,6 +22,15 @@ def test_key(tmp_path): subprocess.run(["gpgconf", "--homedir", str(home), "--kill", "gpg-agent"], capture_output=True) +def _second_key(home): + base = ["gpg", "--homedir", str(home), "--batch", "--pinentry-mode", "loopback", "--passphrase", ""] + subprocess.run([*base, "--quick-generate-key", "bbackup central verifier test", "ed25519", "sign", "1d"], check=True, capture_output=True) + keys = subprocess.run([*base, "--with-colons", "--list-secret-keys"], check=True, capture_output=True, text=True).stdout + fingerprint = [line.split(":")[9] for line in keys.splitlines() if line.startswith("fpr:")][-1] + subprocess.run([*base, "--quick-add-key", fingerprint, "cv25519", "encr", "1d"], check=True, capture_output=True) + return fingerprint + + def test_sealed_kit_authenticates_and_recovers_independent_password(tmp_path, test_key): home, fingerprint = test_key manifest = tmp_path / "checkpoint.jsonl" @@ -57,6 +67,64 @@ def test_nonregular_and_precancelled_inputs_fail_before_gpg(tmp_path): verify_checkpoint(tmp_path / "missing", tmp_path / "sig", "A" * 40, cancel=cancel) +def test_checked_kit_records_indefinite_checkpoint_receipt(tmp_path, monkeypatch): + import json + from pathlib import Path + from types import SimpleNamespace + from bbackup.cloud import CheckpointRecord + from bbackup.models import Configuration, HostBindings, RepositoryBinding + from bbackup.operations import OperationStatus + from bbackup.recovery import create_checked_kit + from bbackup.service import BackupService + + endpoint, bucket, prefix = "https://s3.example.test", "test-bucket", "repo" + password = tmp_path / "password" + password.write_text("disposable password") + password.chmod(0o600) + repository = f"s3:{endpoint}/{bucket}/{prefix}" + config = Configuration.parse({"schema_version": 2, + "repositories": [{"name": "source", "role": "replica"}], + "sources": [], "jobs": []}) + service = BackupService(config, HostBindings(tmp_path / "state", { + "source": RepositoryBinding(repository, password)})) + snapshot, repository_id = "a" * 64, "b" * 64 + service._engine = lambda _name: SimpleNamespace( + base_args=lambda: ["restic", "-r", repository], + check_args=lambda: ["restic", "-r", repository, "check"]) + service.runner.run = lambda plan, **_kwargs: SimpleNamespace( + status=OperationStatus.SUCCEEDED, + stdout_tail=json.dumps({"id": repository_id} if "config" in plan.argv else + {"id": snapshot}).encode()) + operation = SimpleNamespace(operation_id="synthetic-checkpoint", env={}) + def execute(_repository, _argv, *, finalize, **_kwargs): + finalize(operation, None) + return operation, None + service._execute = execute + + class Adapter: + def __init__(self): + self.endpoint_url = endpoint + self.bucket = bucket + self.prefix = prefix + def create_checkpoint(self, path, **_kwargs): + Path(path).write_text("synthetic checkpoint") + return CheckpointRecord(Path(path), "c" * 64, 0, 0, 0, + "2026-10-20T00:00:00Z", None, "source", repository_id, snapshot, + endpoint, bucket, prefix, True) + def seal(_manifest, _password, destination, **_kwargs): + destination.write_bytes(b"synthetic sealed kit") + return {"manifest_sha256": "c" * 64} + monkeypatch.setattr("bbackup.recovery.seal_kit", seal) + result = create_checked_kit(service, Adapter(), "source", snapshot, + tmp_path / "kit.gpg", signer="D" * 40, recipient="E" * 40, + quiescent_window_confirmed=True) + assert result["expires_at"] is None + assert result["repository_id"] == repository_id + with service._receipts() as db: + assert db.execute("SELECT manifest_sha256, expires_at FROM checkpoints").fetchall() == [ + ("c" * 64, None)] + + def test_validsig_does_not_override_revoked_or_expired_status(): from types import SimpleNamespace from bbackup.recovery import _verify_status @@ -66,9 +134,40 @@ def test_validsig_does_not_override_revoked_or_expired_status(): _verify_status(SimpleNamespace(stdout_tail=output, stdout_bytes=len(output)), "A" * 40) +def test_trusted_signer_must_be_primary_fingerprint(): + from types import SimpleNamespace + from bbackup.recovery import _verify_status + subkey, primary = "B" * 40, "A" * 40 + output = (f"[GNUPG:] VALIDSIG {subkey} 2026-10-20 1 0 4 0 22 8 00 {primary}\n").encode() + result = SimpleNamespace(stdout_tail=output, stdout_bytes=len(output)) + _verify_status(result, primary) + with pytest.raises(RecoveryError, match="trusted signer"): + _verify_status(result, subkey) + + +def test_acceptance_publication_never_replaces_a_racing_destination(tmp_path): + from bbackup.recovery import _publish_acceptance_noreplace + + staged = tmp_path / "staged" + staged.mkdir() + (staged / "acceptance.json").write_bytes(b"complete record") + (staged / "acceptance.sig").write_bytes(b"complete signature") + occupied = tmp_path / "accepted" + occupied.mkdir() + (occupied / "independent-record").write_bytes(b"preserve this") + with pytest.raises(RecoveryError, match="already exists"): + _publish_acceptance_noreplace(staged, occupied) + assert (occupied / "independent-record").read_bytes() == b"preserve this" + assert (staged / "acceptance.sig").read_bytes() == b"complete signature" + published = tmp_path / "published" + _publish_acceptance_noreplace(staged, published) + assert not staged.exists() + assert {path.name for path in published.iterdir()} == {"acceptance.json", "acceptance.sig"} + + @pytest.mark.integration @pytest.mark.skipif(shutil.which("restic") is None, reason="requires restic") -def test_historical_repository_and_kit_restore_without_original_ledger(tmp_path, test_key): +def test_historical_repository_and_kit_restore_without_original_ledger(tmp_path, test_key, monkeypatch): import json import io from bbackup.cloud import S3CloudAdapter @@ -78,6 +177,7 @@ def test_historical_repository_and_kit_restore_without_original_ledger(tmp_path, from tests.test_cloud import FakeS3 home, signer = test_key + verifier = _second_key(home) data = tmp_path / "data" data.mkdir() (data / "record.txt").write_text("historical recovery proof") @@ -112,7 +212,7 @@ def get_object(self, **kwargs): checkpoint = adapter.create_checkpoint(manifest, source_repository="local", repository_id=repository_id, source_snapshot_id=captured["snapshot_id"], trusted_quiescent_window_confirmed=True) kit = tmp_path / "kit.gpg" - seal_kit(manifest, password, kit, signer=signer, recipient=signer, home=home) + seal_kit(manifest, password, kit, signer=signer, recipient=verifier, home=home) # Remove only this test's original data, credentials, repository, ledger and manifest. shutil.rmtree(repo) shutil.rmtree(tmp_path / "state") @@ -135,3 +235,73 @@ def get_object(self, **kwargs): clean_service.restore("recovered", captured["snapshot_id"], target) assert (target / str(data / "record.txt").lstrip("/")).read_text() == "historical recovery proof" assert checkpoint.repository_id == repository_id + accepted_dir = tmp_path / "central-acceptance" + accepted = accept_recovery_checkpoint(adapter, kit, accepted_dir, + source_signer=signer, trusted_verifier=verifier, source_repository="local", + repository_id=repository_id, snapshot=captured["snapshot_id"], home=home) + assert accepted["manifest_sha256"] == authenticated["manifest_sha256"] + assert accepted["source_signer"] == signer + assert accepted["verifier"] == verifier + assert accepted["recovery_verified"] is True + assert accepted["protection_qualified"] is False + assert accepted["captured_at"] == checkpoint.captured_at + assert accepted["verified_at"] >= accepted["captured_at"] + assert verify_checkpoint(accepted_dir / "checkpoint.jsonl", accepted_dir / "checkpoint.sig", + signer, home=home) == accepted["manifest_sha256"] + assert verify_recovery_acceptance(accepted_dir, trusted_verifier=verifier, + repository_id=repository_id, snapshot=captured["snapshot_id"], home=home) == accepted + # Even a correctly signed record must satisfy the strict central schema. + from bbackup.recovery import _gpg + for label, raw in ( + ("unknown-field", json.dumps({**accepted, "source_report_trusted": True}).encode()), + ("duplicate-field", json.dumps(accepted).encode()[:-1] + b',"verified_at":"2099-01-01T00:00:00Z"}'), + ("invalid-time", json.dumps({**accepted, "verified_at": "1999-01-01T00:00:00Z"}).encode()), + ): + invalid = tmp_path / label + invalid.mkdir(mode=0o700) + (invalid / "acceptance.json").write_bytes(raw) + _gpg(["--local-user", verifier, "--output", str(invalid / "acceptance.sig"), + "--detach-sign", str(invalid / "acceptance.json")], home=home) + with pytest.raises(RecoveryError): + verify_recovery_acceptance(invalid, trusted_verifier=verifier, + repository_id=repository_id, snapshot=captured["snapshot_id"], home=home) + with pytest.raises(RecoveryError): + verify_recovery_acceptance(accepted_dir, trusted_verifier=signer, + repository_id=repository_id, snapshot=captured["snapshot_id"], home=home) + with pytest.raises(RecoveryError, match="new local directory"): + accept_recovery_checkpoint(adapter, kit, accepted_dir, + source_signer=signer, trusted_verifier=verifier, source_repository="local", + repository_id=repository_id, snapshot=captured["snapshot_id"], home=home) + with monkeypatch.context() as patch: + def unavailable_proof(*args, **kwargs): + raise RecoveryError("Reconstructed repository verification failed") + patch.setattr("bbackup.recovery.verify_reconstructed_repository", unavailable_proof) + unproven_dir = tmp_path / "unproven-acceptance" + with pytest.raises(RecoveryError, match="verification failed"): + accept_recovery_checkpoint(adapter, kit, unproven_dir, + source_signer=signer, trusted_verifier=verifier, source_repository="local", + repository_id=repository_id, snapshot=captured["snapshot_id"], home=home) + assert not unproven_dir.exists() + for label, captured_at, expires_at in ( + ("future", "2099-01-01T00:00:00Z", "2100-01-01T00:00:00Z"), + ("expired", "2000-01-01T00:00:00Z", "2001-01-01T00:00:00Z"), + ): + forged_manifest = tmp_path / f"{label}.jsonl" + original = (opened / "checkpoint.jsonl").read_bytes().splitlines(keepends=True) + header = json.loads(original[0]) + header["captured_at"], header["expires_at"] = captured_at, expires_at + original[0] = json.dumps(header, sort_keys=True, separators=(",", ":")).encode() + b"\n" + forged_manifest.write_bytes(b"".join(original)) + forged_kit = tmp_path / f"{label}.gpg" + seal_kit(forged_manifest, opened / "repository.password", forged_kit, + signer=signer, recipient=verifier, home=home) + refused_dir = tmp_path / f"{label}-acceptance" + with pytest.raises(RecoveryError, match="future-dated or expired"): + accept_recovery_checkpoint(adapter, forged_kit, refused_dir, + source_signer=signer, trusted_verifier=verifier, source_repository="local", + repository_id=repository_id, snapshot=captured["snapshot_id"], home=home) + assert not refused_dir.exists() + (accepted_dir / "acceptance.json").write_text("tampered") + with pytest.raises(RecoveryError): + verify_recovery_acceptance(accepted_dir, trusted_verifier=verifier, + repository_id=repository_id, snapshot=captured["snapshot_id"], home=home) diff --git a/tests/test_resource_limits.py b/tests/test_resource_limits.py new file mode 100644 index 0000000..b178b5c --- /dev/null +++ b/tests/test_resource_limits.py @@ -0,0 +1,61 @@ +import pytest + +from bbackup.models import ResourceBudget +from bbackup.resource_limits import ResourceLimitError, ResourceMonitor + + +def test_dedicated_private_mount_and_observed_reserve(tmp_path, monkeypatch): + staging = tmp_path / "staging" + state = tmp_path / "state" + staging.mkdir(mode=0o700) + state.mkdir(mode=0o700) + volumes = {staging: (1000, 700), state: (10000, 9000)} + monkeypatch.setattr("bbackup.resource_limits.os.path.ismount", lambda path: path == staging) + monkeypatch.setattr("bbackup.resource_limits._volume", lambda path: volumes[path]) + monitor = ResourceMonitor(staging, state, ResourceBudget(1000, 100)) + monitor.check(500) + assert monitor.result()["staging_capacity_bounded"] is True + assert monitor.result()["free_space_reserve_hard_guaranteed"] is False + assert monitor.result()["usage_attributed_to_this_operation"] is False + with pytest.raises(ResourceLimitError, match="reserve"): + monitor.check(601) + assert monitor.result()["staging_minimum_free_bytes"] == 700 + volumes[state] = (10000, 50) + with pytest.raises(ResourceLimitError, match="reserve"): + monitor.check() + monitor.close() + + +def test_remote_staging_rejects_shared_or_oversized_mount(tmp_path, monkeypatch): + staging = tmp_path / "staging" + staging.mkdir(mode=0o700) + monkeypatch.setattr("bbackup.resource_limits._volume", lambda path: (2000, 1900)) + with pytest.raises(ResourceLimitError, match="dedicated"): + ResourceMonitor(staging, tmp_path, ResourceBudget(1000, 100)) + monkeypatch.setattr("bbackup.resource_limits.os.path.ismount", lambda path: path == staging) + with pytest.raises(ResourceLimitError, match="capacity"): + ResourceMonitor(staging, tmp_path, ResourceBudget(1000, 100)) + (staging / "unowned").write_text("do not remove") + with pytest.raises(ResourceLimitError, match="empty"): + ResourceMonitor(staging, tmp_path, ResourceBudget(3000, 100)) + assert (staging / "unowned").exists() + + +def test_staging_lease_is_nonblocking_across_independent_state_roots(tmp_path, monkeypatch): + staging = tmp_path / "staging" + first_state = tmp_path / "first-state" + second_state = tmp_path / "second-state" + for path in (staging, first_state, second_state): + path.mkdir(mode=0o700) + monkeypatch.setattr("bbackup.resource_limits.os.path.ismount", lambda path: path == staging) + monkeypatch.setattr("bbackup.resource_limits._volume", lambda _path: (1000, 900)) + with ResourceMonitor(staging, first_state, ResourceBudget(1000, 100)) as first: + lock = staging / ".bbackup-staging.lock" + original_inode = lock.stat().st_ino + with pytest.raises(ResourceLimitError, match="busy"): + ResourceMonitor(staging, second_state, ResourceBudget(1000, 100)) + first.check() + assert lock.stat().st_ino == original_inode + with ResourceMonitor(staging, second_state, ResourceBudget(1000, 100)) as second: + second.check() + assert lock.stat().st_ino == original_inode diff --git a/tests/test_scheduling.py b/tests/test_scheduling.py index 374c70d..2eb4393 100644 --- a/tests/test_scheduling.py +++ b/tests/test_scheduling.py @@ -3,7 +3,7 @@ import pytest from bbackup.models import Job -from bbackup.scheduling import install_user_units, job_units, unit_templates +from bbackup.scheduling import install_user_units, job_units, system_job_units, unit_templates def test_unit_templates_are_disabled_and_use_secure_daily_defaults() -> None: @@ -57,6 +57,57 @@ def test_job_renderer_keeps_custom_schedule_and_shared_service_hardening() -> No assert "NoNewPrivileges=yes" in units["bbackup-nightly.service"] +def test_root_renderer_uses_shared_production_command_and_requires_trusted_inputs(monkeypatch): + job = Job(name="daily", repository="primary", sources=("files",)) + paths = {"executable": Path("/usr/bin/bbackup"), + "config": Path("/etc/bbackup/config.json"), + "bindings": Path("/etc/bbackup/bindings.json")} + monkeypatch.setattr("bbackup.scheduling.os.geteuid", lambda: 1000) + with pytest.raises(ValueError, match="root identity"): + system_job_units(job, **paths) + monkeypatch.setattr("bbackup.scheduling.os.geteuid", lambda: 0) + checked = [] + def root_owned(path, **kwargs): + checked.append((path, kwargs)) + monkeypatch.setattr("bbackup.scheduling._root_owned_path", root_owned) + units = system_job_units(job, **paths) + service = units["bbackup-daily.service"] + assert "User=root\nGroup=root" in service + assert "UMask=0077" in service and "KillMode=control-group" in service + assert " production --config " in service and "--bindings" in service + assert "sudo" not in service + assert "Persistent=true" in units["bbackup-daily.timer"] + assert checked[-1] == (paths["bindings"], {"private": True}) + + +@pytest.mark.parametrize("covered", [False, True]) +@pytest.mark.parametrize("system", [False, True]) +def test_covered_timers_preserve_reviewed_temporary_source_view(monkeypatch, covered, system): + scope = {"schema_version": 1, "approved_roots": ["system"], "require_privileged": system, + "filesystem_boundaries": "explicit", "metadata_expectations": { + key: True for key in ("numeric_owner", "mode", "acl", "xattr", "links")}} + job = Job(name="whole-system", repository="primary", sources=("files",), + coverage_policy=scope if covered else None) + monkeypatch.setattr("bbackup.scheduling.os.geteuid", lambda: 0) + monkeypatch.setattr("bbackup.scheduling._root_owned_path", lambda *args, **kwargs: None) + render = system_job_units if system else job_units + unit = render(job, executable=Path("/usr/bin/bbackup"), + config=Path("/etc/bbackup/config.json"), + bindings=Path("/etc/bbackup/bindings.json"))["bbackup-whole-system.service"] + assert ("PrivateTmp=no\n" if covered else "PrivateTmp=yes\n") in unit + assert "NoNewPrivileges=yes\n" in unit + assert "UMask=0077\n" in unit + + +def test_root_preflight_rejects_user_controlled_ancestors_and_bindings(tmp_path, monkeypatch): + from bbackup.scheduling import _root_owned_path + monkeypatch.setattr("bbackup.scheduling.os.geteuid", lambda: 0) + candidate = tmp_path / "config.json" + candidate.write_text("{}") + with pytest.raises(ValueError, match="root controlled"): + _root_owned_path(candidate) + + def test_user_install_requires_explicit_activation_and_preserves_existing_units(tmp_path, monkeypatch): monkeypatch.setattr(Path, "home", lambda: tmp_path) monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) diff --git a/tests/test_service.py b/tests/test_service.py index 5ae75ec..8a6a3e2 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,20 +1,31 @@ import json import gzip +import hashlib +import os import shutil import sqlite3 +import subprocess +import sys from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace import pytest -from bbackup.models import Configuration, HostBindings, RepositoryBinding +from bbackup.models import Configuration, HostBindings, RepositoryBinding, Source, SourceBinding +from bbackup.enrollment import policy_lease, write_bytes +from bbackup.linux_coverage import (CoverageCheck, InventoryPaths, check_coverage, + discover_coverage, review_coverage, + sqlite_file_identity) from bbackup.operations import ( OperationPlan, OperationRun, OperationStatus, + ReconciliationOutcome, RepositoryConflictError, repository_key, ) -from bbackup.service import BackupService, ReplicaJobError, ServiceError +from bbackup.service import BackupService, ReplicaJobError, ResourceBudgetExceeded, ServiceError @pytest.fixture @@ -53,6 +64,985 @@ def _run(plan, *, status=OperationStatus.SUCCEEDED, exit_code=0, stdout=b""): ) +def test_covered_capture_requires_enrollment_and_exact_root_mapping(tmp_path, monkeypatch): + data = tmp_path / "data" + data.mkdir() + password = data / ".credential" + password.write_text("private fixture credential") + password.chmod(0o600) + scope = {"schema_version": 1, "approved_roots": ["data"], + "require_privileged": False, "filesystem_boundaries": "explicit", + "metadata_expectations": {name: True for name in + ("numeric_owner", "mode", "acl", "xattr", "links")}} + config = Configuration.parse({"schema_version": 3, + "repositories": [{"name": "primary"}], + "sources": [{"name": "files", "kind": "files"}], + "jobs": [{"name": "daily", "repository": "primary", "sources": ["files"], + "coverage_policy": scope}]}) + bindings = HostBindings(tmp_path / "state", { + "primary": RepositoryBinding(str(tmp_path / "repository"), password), + }, source_bindings={"files": SourceBinding(paths=(str(data),))}, schema_version=3, + coverage_declarations={"daily": tmp_path / "declaration.json"}) + service = BackupService(config, bindings) + with pytest.raises(ServiceError, match="exact enrolled"): + service.capture("daily") + service = BackupService(config, bindings, policy_bytes=b'{"policy":1}', + binding_bytes=b'{"binding":1}') + calls = [] + def check(_path, _policy, **kwargs): + calls.append(kwargs) + return CoverageCheck("complete", (), "a" * 64, "b" * 64, {}, + (("data", data, "files", "files"),)) + monkeypatch.setattr("bbackup.linux_coverage.check_coverage", check) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, stdout=(json.dumps({"message_type": "summary", "snapshot_id": "a" * 64}) + "\n").encode())) + assert service.capture("daily")["snapshot_id"] == "a" * 64 + assert calls[0]["policy_bytes"] == b'{"policy":1}' + monkeypatch.setattr("bbackup.linux_coverage.check_coverage", lambda *_args, **_kwargs: + CoverageCheck("complete", (), "a" * 64, "b" * 64, {}, + (("data", tmp_path / "other", "files", "files"),))) + with pytest.raises(ServiceError, match="root mappings"): + service.capture("daily") + + +def test_covered_postgresql_requires_manual_quiet_window_and_exact_sources(tmp_path, monkeypatch): + from bbackup.models import DatabaseBinding + + recovery = tmp_path / "recovery" + recovery.mkdir() + material = recovery / "roles.sql" + material.write_text("reviewed roles") + material.chmod(0o600) + password = tmp_path / "restic-password" + password.write_text("fixture-password") + password.chmod(0o600) + service_file = tmp_path / "pg_service.conf" + service_file.write_text("[local]\nhost=/var/run/postgresql\nuser=postgres\n") + service_file.chmod(0o600) + scope = {"schema_version": 1, "approved_roots": ["recovery_root", "app_root"], + "require_privileged": False, "filesystem_boundaries": "explicit", + "metadata_expectations": {name: True for name in + ("numeric_owner", "mode", "acl", "xattr", "links")}} + config = Configuration.parse({"schema_version": 3, + "repositories": [{"name": "primary"}], + "sources": [{"name": "files", "kind": "files"}, + {"name": "database", "kind": "postgresql"}], + "jobs": [{"name": "daily", "repository": "primary", + "sources": ["files", "database"], "coverage_policy": scope}]}) + bindings = HostBindings(tmp_path / "state", { + "primary": RepositoryBinding(str(tmp_path / "repository"), password)}, + database_bindings={"pg_local": DatabaseBinding( + "postgresql", service_file=service_file, service_name="local")}, + source_bindings={"files": SourceBinding(paths=(str(recovery),)), + "database": SourceBinding(database="app", connection="pg_local")}, + schema_version=3, coverage_declarations={"daily": tmp_path / "declaration.json"}) + captured = BackupService(config, bindings, policy_bytes=b"policy", binding_bytes=b"binding") + reviewed = {"source_id": "database", "connection": "pg_local", "database": "app", + "root_id": "app_root", "cluster_id": "primary", "baseline_digest": "a" * 64, + "data_directory_identity": {"device": 1}, + "recovery_artifacts": [{"path": str(material), "root_id": "recovery_root", + "sha256": hashlib.sha256(material.read_bytes()).hexdigest()}]} + observed = [] + + def check(_declaration, _policy, **kwargs): + context = kwargs["postgresql_context"] + observed.append(context.quiet_administrative_window) + if not context.quiet_administrative_window: + return CoverageCheck("incomplete", ("postgresql_quiet_administrative_window_required",), + None, None, {}) + return CoverageCheck("complete", (), "a" * 64, "b" * 64, {}, + (("recovery_root", recovery, "files", "files"),), + approved_postgresql=(reviewed,)) + + monkeypatch.setattr("bbackup.linux_coverage.check_coverage", check) + with pytest.raises(ServiceError, match="not ready"): + captured.capture("daily") + + def stop_before_export(*_args, **kwargs): + assert kwargs["coverage_postgresql"]["database"]["database"] == "app" + assert callable(kwargs["validate_coverage"]) + raise RuntimeError("manual PostgreSQL capture reached export") + + monkeypatch.setattr("bbackup.service.capture_sources", stop_before_export) + with pytest.raises(RuntimeError, match="reached export"): + captured.capture("daily", quiet_administrative_window_confirmed=True) + assert observed == [False, True] + + checks = 0 + + def drifts_after_export(*args, **kwargs): + nonlocal checks + checks += 1 + if checks == 1: + return check(*args, **kwargs) + return CoverageCheck("drift", ("postgresql_reconstruction_evidence_stale",), + "a" * 64, "changed", {}) + + def stage_then_recheck(*_args, **kwargs): + kwargs["validate_coverage"]() + pytest.fail("drift reached Restic dispatch") + + monkeypatch.setattr("bbackup.linux_coverage.check_coverage", drifts_after_export) + monkeypatch.setattr("bbackup.service.capture_sources", stage_then_recheck) + with pytest.raises(ServiceError, match="changed during capture"): + captured.capture("daily", quiet_administrative_window_confirmed=True) + assert checks == 2 + + +@pytest.mark.integration +@pytest.mark.skipif(shutil.which("restic") is None, reason="requires restic") +def test_covered_exact_exclusion_is_absent_from_real_restored_snapshot(tmp_path, monkeypatch): + data = tmp_path / "data[fixture]" + data.mkdir() + (data / "keep.txt").write_text("keep") + excluded = data / "omit*.txt" + excluded.write_text("excluded secret") + password = tmp_path / "password" + password.write_text("fixture-password") + password.chmod(0o600) + scope = {"schema_version": 1, "approved_roots": ["data"], + "require_privileged": False, "filesystem_boundaries": "explicit", + "metadata_expectations": {name: True for name in + ("numeric_owner", "mode", "acl", "xattr", "links")}} + config = Configuration.parse({"schema_version": 3, + "repositories": [{"name": "primary"}], + "sources": [{"name": "files", "kind": "files"}], + "jobs": [{"name": "daily", "repository": "primary", "sources": ["files"], + "coverage_policy": scope}]}) + bindings = HostBindings(tmp_path / "state", { + "primary": RepositoryBinding(str(tmp_path / "repository"), password), + }, source_bindings={"files": SourceBinding(paths=(str(data),))}, schema_version=3, + coverage_declarations={"daily": tmp_path / "declaration.json"}) + service = BackupService(config, bindings, policy_bytes=b'{"policy":1}', + binding_bytes=b'{"binding":1}') + monkeypatch.setattr("bbackup.linux_coverage.check_coverage", lambda *_args, **_kwargs: + CoverageCheck("complete", (), "a" * 64, "b" * 64, {}, + (("data", data, "files", "files"),), (excluded,))) + service.initialize("primary") + snapshot = service.capture("daily")["snapshot_id"] + restored = tmp_path / "restored" + service.restore("primary", snapshot, restored) + tree = restored / str(data).lstrip("/") + assert (tree / "keep.txt").read_text() == "keep" + assert not (tree / "omit*.txt").exists() + + +@pytest.mark.integration +@pytest.mark.skipif(shutil.which("restic") is None, reason="requires restic") +def test_covered_sqlite_export_restores_with_reviewed_metadata_recipe(tmp_path, monkeypatch): + data = tmp_path / "data[fixture]" + data.mkdir() + (data / "keep.txt").write_text("sibling survives\n") + database_path = data / "database [review]*?.sqlite" + writer = sqlite3.connect(database_path) + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("CREATE TABLE committed (value TEXT)") + writer.execute("INSERT INTO committed VALUES ('committed row')") + writer.commit() + writer.execute("INSERT INTO committed VALUES ('uncommitted row')") + wal_path = Path(str(database_path) + "-wal") + assert wal_path.is_file() + try: + os.setxattr(database_path, "user.coverage", b"restore-me") + except (AttributeError, OSError): + writer.close() + pytest.skip("Fixture filesystem cannot store SQLite xattrs") + source_identity = sqlite_file_identity(database_path) + source_metadata_json = json.dumps(source_identity, sort_keys=True) + + def write_private(path, value): + path.write_text(json.dumps(value, sort_keys=True), encoding="utf-8") + path.chmod(0o600) + return path + + def recipe(name, procedure_text): + procedure = tmp_path / f"{name}.py" + procedure.write_text(procedure_text, encoding="utf-8") + procedure.chmod(0o600) + digest = hashlib.sha256(procedure.read_bytes()).hexdigest() + evidence = write_private(tmp_path / f"{name}.evidence.json", { + "schema_version": 1, "recipe_id": name, "recipe_sha256": digest, + "environment": "isolated-local-fixture", + "checks": {"procedure_executed": True, "result_verified": True}, + }) + return ({"reason": "rebuild covered state", "consequence": "run on recovery", + "recipe_id": name, "recipe_sha256": digest}, + {"recipe_id": name, "recipe_sha256": digest, + "procedure_path": str(procedure), "evidence_path": str(evidence), + "evidence_sha256": hashlib.sha256(evidence.read_bytes()).hexdigest()}, + procedure) + + metadata_recipe_body = ( + "import base64, json, os, sys\n" + f"metadata = json.loads({source_metadata_json!r})\n" + "path = sys.argv[1]\n" + "os.chown(path, metadata['uid'], metadata['gid'])\n" + "os.chmod(path, metadata['mode'])\n" + "for item in metadata['xattrs']:\n" + " os.setxattr(path, item['name'], base64.b64decode(item['value_base64']))\n" + ) + app_recipe, app_evidence, recipe_path = recipe("sqlite_rebuild", metadata_recipe_body) + package_recipe, package_evidence, _ = recipe( + "package_rebuild", "from pathlib import Path\nPath(__file__).read_text()\n") + inventory_root = tmp_path / "inventory" + inventory_root.mkdir() + passwd = inventory_root / "passwd" + passwd.write_text(f"fixture:x:1000:1000::{data}:/bin/sh\n") + group = inventory_root / "group" + group.write_text("fixture:x:1000:\n") + mountinfo = inventory_root / "mountinfo" + mountinfo.write_text("1 1 0:1 / / rw - ext4 /dev/fixture rw\n") + packages = inventory_root / "packages" + packages.write_text("Package: fixture\n") + units = inventory_root / "units" + units.mkdir() + (units / "fixture.service").write_text("[Service]\n") + inventory = InventoryPaths(passwd, group, mountinfo, packages, units) + scope = {"schema_version": 1, "approved_roots": ["data", "database"], + "require_privileged": False, "filesystem_boundaries": "explicit", + "metadata_expectations": {name: True for name in + ("numeric_owner", "mode", "acl", "xattr", "links")}} + declaration = write_private(tmp_path / "declaration.json", { + "schema_version": 1, + "roots": [{"id": "data", "path": str(data), "source_id": "files", "adapter": "files"}, + {"id": "database", "path": str(database_path), "source_id": "db", "adapter": "sqlite"}], + "exclusions": [], "mount_decisions": [], "identity_decisions": [], + "applications": [{"id": "fixture", "unit": "fixture", "source_id": "db", + "state_roots": [str(database_path)], "startup_recipe": app_recipe}], + "package_reconstruction": package_recipe, + "reconstruction_evidence": [app_evidence, package_evidence], + }) + policy_bytes, binding_bytes = b'{"policy":1}', b'{"binding":1}' + state = tmp_path / "state" + write_bytes(state, policy_bytes, binding_bytes) + with policy_lease(state, create=True): + pass + discovered = discover_coverage(declaration, scope, state=state, + policy_bytes=policy_bytes, binding_bytes=binding_bytes, inventory=inventory) + assert discovered["coverage_status"] == "complete" + reviewed = review_coverage(declaration, scope, discovered["candidate_digest"], + state=state, policy_bytes=policy_bytes, binding_bytes=binding_bytes, inventory=inventory) + assert reviewed["coverage_status"] == "complete" + + password = tmp_path / "password" + password.write_text("fixture-password") + password.chmod(0o600) + config = Configuration.parse({"schema_version": 3, + "repositories": [{"name": "primary"}], + "sources": [{"name": "files", "kind": "files"}, {"name": "db", "kind": "sqlite"}], + "jobs": [{"name": "daily", "repository": "primary", "sources": ["files", "db"], + "coverage_policy": scope}]}) + bindings = HostBindings(state, { + "primary": RepositoryBinding(str(tmp_path / "repository"), password), + }, source_bindings={"files": SourceBinding(paths=(str(data),)), + "db": SourceBinding(paths=(str(database_path),))}, + schema_version=3, coverage_declarations={"daily": declaration}) + service = BackupService(config, bindings, policy_bytes=policy_bytes, + binding_bytes=binding_bytes) + real_check = check_coverage + monkeypatch.setattr("bbackup.linux_coverage.check_coverage", + lambda declaration_path, policy, **kwargs: real_check( + declaration_path, policy, inventory=inventory, **kwargs)) + service.initialize("primary") + snapshot = service.capture("daily")["snapshot_id"] + writer.rollback() + writer.close() + + restored = tmp_path / "restored" + service.restore("primary", snapshot, restored) + restored_manifest, = restored.rglob("bbackup-capture.json") + manifest = json.loads(restored_manifest.read_text()) + assert manifest["schema_version"] == 2 + entries = [item for item in manifest["exports"] if item["source"] == "db"] + assert len(entries) == 1 + entry = entries[0] + assert entry["coverage_root_id"] == "database" + assert entry["original_path"] == str(database_path) + assert entry["baseline_digest"] == reviewed["baseline_digest"] + assert entry["recipe_digest"] == app_recipe["recipe_sha256"] + assert entry["reviewed_identity"] == source_identity + export = restored / entry["path"].lstrip("/") + assert hashlib.sha256(export.read_bytes()).hexdigest() == entry["export_sha256"] + with sqlite3.connect(export) as database: + assert database.execute("PRAGMA quick_check").fetchone() == ("ok",) + assert database.execute("SELECT value FROM committed").fetchall() == [ + ("committed row",)] + restored_data = restored / str(data).lstrip("/") + assert (restored_data / "keep.txt").read_text() == "sibling survives\n" + restored_raw = restored_data / database_path.name + assert not restored_raw.exists() + for suffix in ("-wal", "-shm", "-journal"): + assert not Path(str(restored_raw) + suffix).exists() + assert not (restored / str(wal_path).lstrip("/")).exists() + + fresh = tmp_path / "fresh-database.sqlite" + shutil.copyfile(export, fresh) + subprocess.run([sys.executable, str(recipe_path), str(fresh)], check=True) + with sqlite3.connect(fresh) as database: + assert database.execute("PRAGMA quick_check").fetchone() == ("ok",) + assert database.execute("SELECT value FROM committed").fetchall() == [ + ("committed row",)] + reconstructed = sqlite_file_identity(fresh) + for key in ("uid", "gid", "mode", "links", "xattrs"): + assert reconstructed[key] == source_identity[key] + + direct_scope = {**scope, "approved_roots": ["data", "wal", "database"]} + direct_declaration_raw = json.loads(declaration.read_text()) + direct_declaration_raw["roots"].insert(1, {"id": "wal", "path": str(wal_path), + "source_id": "files", "adapter": "files"}) + direct_declaration = write_private(tmp_path / "direct-wal-declaration.json", + direct_declaration_raw) + direct_candidate = discover_coverage(direct_declaration, direct_scope, state=state, + policy_bytes=policy_bytes, binding_bytes=binding_bytes, inventory=inventory) + assert direct_candidate["coverage_status"] == "incomplete" + assert "sqlite_artifact_explicitly_mapped" in direct_candidate["reasons"] + direct_config = Configuration.parse({"schema_version": 3, + "repositories": [{"name": "primary"}], + "sources": [{"name": "files", "kind": "files"}, {"name": "db", "kind": "sqlite"}], + "jobs": [{"name": "daily", "repository": "primary", "sources": ["files", "db"], + "coverage_policy": direct_scope}]}) + direct_bindings = HostBindings(state, { + "primary": RepositoryBinding(str(tmp_path / "repository"), password), + }, source_bindings={"files": SourceBinding(paths=(str(data), str(wal_path))), + "db": SourceBinding(paths=(str(database_path),))}, + schema_version=3, coverage_declarations={"daily": direct_declaration}) + direct_service = BackupService(direct_config, direct_bindings, + policy_bytes=policy_bytes, binding_bytes=binding_bytes) + monkeypatch.setattr(direct_service.runner, "run", + lambda *_args, **_kwargs: pytest.fail("unsafe WAL source reached Restic")) + with pytest.raises(ServiceError, match="Covered capture is not ready"): + direct_service.capture("daily") + + def mismatched_review(*args, **kwargs): + accepted = real_check(*args, inventory=inventory, **kwargs) + return replace(accepted, approved_sqlite=()) + monkeypatch.setattr("bbackup.linux_coverage.check_coverage", mismatched_review) + with pytest.raises(ServiceError, match="SQLite sources differ"): + service.capture("daily") + + +def _remote_service(tmp_path, *, with_replica=False): + data = tmp_path / "large-source" + with data.open("wb") as stream: + stream.truncate(2 * 1024**2) + password = tmp_path / "password" + password.write_text("fixture-secret") + password.chmod(0o600) + staging = tmp_path / "staging" + staging.mkdir(mode=0o700) + config = Configuration.parse({ + "schema_version": 2, + "repositories": [{"name": "primary", "role": "remote-primary"}, + *([{"name": "copy", "role": "replica"}] if with_replica else [])], + "sources": [{"name": "files", "kind": "files", "paths": [str(data)]}], + "jobs": [{"name": "daily", "repository": "primary", "sources": ["files"], + "resource_budget": {"schema_version": 1, "staging_max_bytes": 1024**3, + "minimum_free_bytes": 64 * 1024**2, + "cache_max_bytes": 0}}], + }) + bindings = HostBindings(tmp_path / "state", { + "primary": RepositoryBinding("s3:https://s3.example.test/bucket/repo", password), + **({"copy": RepositoryBinding(str(tmp_path / "copy"), password)} if with_replica else {}), + }, staging_dir=staging) + return BackupService(config, bindings), data, staging + + +def _remote_sqlite_service(tmp_path, monkeypatch): + database_path = tmp_path / "live.sqlite" + with sqlite3.connect(database_path) as database: + database.execute("CREATE TABLE data(value TEXT)") + database.execute("INSERT INTO data VALUES ('captured')") + password = tmp_path / "password" + password.write_text("fixture-secret") + password.chmod(0o600) + staging = tmp_path / "staging" + staging.mkdir(mode=0o700) + config = Configuration.parse({ + "schema_version": 2, + "repositories": [{"name": "primary", "role": "remote-primary"}], + "sources": [{"name": "db", "kind": "sqlite", "paths": [str(database_path)]}], + "jobs": [{"name": "daily", "repository": "primary", "sources": ["db"], + "resource_budget": {"schema_version": 1, "staging_max_bytes": 1024**3, + "minimum_free_bytes": 1, "cache_max_bytes": 0}}], + }) + bindings = HostBindings(tmp_path / "state", { + "primary": RepositoryBinding("s3:https://s3.example.test/bucket/repo", password), + }, staging_dir=staging) + monkeypatch.setattr("bbackup.resource_limits.os.path.ismount", lambda path: path == staging) + monkeypatch.setattr("bbackup.resource_limits._volume", lambda _path: (1024**3, 1024**3 - 10000)) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fixture-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + return BackupService(config, bindings), database_path, staging + + +def _retained_capture(service, staging): + stages = [path for path in staging.iterdir() if path.name.startswith("capture-")] + assert len(stages) == 1 + token = stages[0].name.removeprefix("capture-") + record = json.loads((service.state / "capture-artifacts" / f"{token}.json").read_text()) + return token, stages[0], record + + +def test_remote_failed_upload_keeps_exact_export_after_database_changes(tmp_path, monkeypatch): + service, database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, status=OperationStatus.FAILED, exit_code=1)) + with pytest.raises(ServiceError, match="retained capture token"): + service.capture("daily") + token, stage, record = _retained_capture(service, staging) + exported = stage / "db.sqlite" + digest = hashlib.sha256(exported.read_bytes()).hexdigest() + assert record["exports"][0]["sha256"] == digest + assert record["exports"][0]["size"] == exported.stat().st_size + with sqlite3.connect(database_path) as database: + database.execute("UPDATE data SET value='changed after failure'") + assert hashlib.sha256(exported.read_bytes()).hexdigest() == digest + with sqlite3.connect(exported) as database: + assert database.execute("SELECT value FROM data").fetchone() == ("captured",) + inspected = [] + + def inspect(plan, **_kwargs): + inspected.append(plan.argv) + assert "--no-lock" in plan.argv + if "locks" in plan.argv: + return _run(plan) + assert "snapshots" in plan.argv + return _run(plan, stdout=b"[]") + + monkeypatch.setattr(service.runner, "run", inspect) + result = service.reconcile_run("primary", token, ReconciliationOutcome.FAILED, quiescent_window_confirmed=True) + assert result["retained_exports"] is True + assert exported.exists() and inspected + monkeypatch.setattr(service.runner, "run", lambda *_args, **_kwargs: pytest.fail( + "retained stage triggered another upload")) + with pytest.raises(ResourceBudgetExceeded): + service.capture("daily") + + +def test_remote_pre_dispatch_failure_retains_completed_export_without_ledger_row(tmp_path, monkeypatch): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + service.sources = (*service.sources, Source("missing", "sqlite", (str(tmp_path / "missing"),))) + original_job = service._job("daily") + monkeypatch.setattr(service, "_job", lambda _name: replace(original_job, sources=("db", "missing"))) + monkeypatch.setattr(service.runner, "run", lambda *_args, **_kwargs: pytest.fail( + "pre-dispatch failure reached upload")) + with pytest.raises(ServiceError, match="retained capture token"): + service.capture("daily") + token, stage, record = _retained_capture(service, staging) + assert service.ledger.inspect(repository_key(service._identity("primary")), token) is None + assert record["exports"][0]["name"] == "db.sqlite" + assert (stage / "db.sqlite").exists() + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, stdout=b"" if "locks" in plan.argv else b"[]")) + result = service.reconcile_run("primary", token, ReconciliationOutcome.ABANDONED, quiescent_window_confirmed=True) + assert result["retained_exports"] is True and (stage / "db.sqlite").exists() + listing = service.retained_exports("primary") + assert listing == [{"repository": "primary", "operation_id": token, + "phase": "prepared", "reviewed_outcome": "abandoned", + "completed_export_count": 1, + "completed_export_bytes": (stage / "db.sqlite").stat().st_size, + "artifact_record_sha256": result["artifact_record_sha256"], + "file_hashes_verified": False}] + assert str(stage) not in str(listing) + monkeypatch.delenv("AWS_ACCESS_KEY_ID") + monkeypatch.delenv("AWS_SECRET_ACCESS_KEY") + disposed = service.discard_exports("primary", token, listing[0]["artifact_record_sha256"]) + assert disposed["discarded_export_count"] == 1 + assert not stage.exists() and service.retained_exports("primary") == [] + assert {path.name for path in staging.iterdir()} == {".bbackup-staging.lock"} + + +def test_remote_discard_refuses_unknown_outcome_stale_digest_and_foreign_file(tmp_path, monkeypatch): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, status=OperationStatus.FAILED, exit_code=1)) + with pytest.raises(ServiceError): + service.capture("daily") + token, stage, _record = _retained_capture(service, staging) + before = service.retained_exports("primary")[0]["artifact_record_sha256"] + with pytest.raises(ServiceError, match="reconciled"): + service.discard_exports("primary", token, before) + assert (stage / "db.sqlite").exists() + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, stdout=b"" if "locks" in plan.argv else b"[]")) + service.reconcile_run("primary", token, ReconciliationOutcome.FAILED, quiescent_window_confirmed=True) + current = service.retained_exports("primary")[0]["artifact_record_sha256"] + assert current != before + with pytest.raises(ServiceError, match="refused"): + service.discard_exports("primary", token, before) + foreign = stage / "foreign" + foreign.write_text("keep") + with pytest.raises(ServiceError, match="refused"): + service.discard_exports("primary", token, current) + assert foreign.read_text() == "keep" and (stage / "db.sqlite").exists() + foreign.unlink() + service.discard_exports("primary", token, current) + assert not stage.exists() + + +def test_remote_restart_orphan_does_not_dispatch_or_discard(tmp_path, monkeypatch): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda _plan, **_kwargs: (_ for _ in ()).throw( + RuntimeError("fixture process lost"))) + with pytest.raises(ServiceError, match="retained capture token"): + service.capture("daily") + token, stage, _record = _retained_capture(service, staging) + restarted = BackupService(service.config, service.bindings) + monkeypatch.setattr(restarted.runner, "run", lambda *_args, **_kwargs: pytest.fail( + "restart replayed uncertain upload")) + with pytest.raises(ResourceBudgetExceeded): + restarted.capture("daily") + assert restarted.ledger.inspect(repository_key(restarted._identity("primary")), token)[1] is OperationStatus.UNCERTAIN + assert (stage / "db.sqlite").exists() + + +def test_remote_receipt_failure_retains_completed_export(tmp_path, monkeypatch): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + snapshot = "a" * 64 + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, stdout=(json.dumps({"message_type": "summary", "snapshot_id": snapshot}) + "\n").encode())) + monkeypatch.setattr(service, "_receipts", lambda: (_ for _ in ()).throw(sqlite3.OperationalError( + "fixture receipt failure"))) + with pytest.raises(ServiceError, match="retained capture token"): + service.capture("daily") + _, stage, record = _retained_capture(service, staging) + assert record["phase"] == "ready" + assert (stage / "db.sqlite").exists() + + +def test_remote_success_reconciliation_repairs_receipt_and_cleans_exact_stage(tmp_path, monkeypatch): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, status=OperationStatus.FAILED, exit_code=1)) + with pytest.raises(ServiceError): + service.capture("daily") + token, stage, record = _retained_capture(service, staging) + snapshot = "b" * 64 + manifest = (stage / record["manifest"]["name"]).read_bytes() + exported = (stage / "db.sqlite").read_bytes() + inspected = [] + + def inspect(plan, **kwargs): + inspected.append(plan.argv) + assert "--no-lock" in plan.argv + if "locks" in plan.argv: + return _run(plan) + if "snapshots" in plan.argv: + return _run(plan, stdout=json.dumps([{"id": snapshot, + "tags": [f"bbackup-capture={token}", "bbackup-job=daily"]}]).encode()) + assert "dump" in plan.argv + payload = exported if plan.argv[-1].endswith("db.sqlite") else manifest + kwargs["on_stdout_chunk"](payload) + return replace(_run(plan, stdout=payload), stdout_tail=b"") + + monkeypatch.setattr(service.runner, "run", inspect) + result = service.reconcile_run("primary", token, ReconciliationOutcome.SUCCEEDED, quiescent_window_confirmed=True) + assert result["snapshot_id"] == snapshot + assert {"locks", "snapshots", "dump"}.issubset({word for argv in inspected for word in argv}) + assert [argv[-2:] for argv in inspected if "locks" in argv] == [("list", "locks")] * 2 + assert not stage.exists() + assert {path.name for path in staging.iterdir()} == {".bbackup-staging.lock"} + assert service.snapshots()[0]["snapshot_id"] == snapshot + assert service.ledger.inspect(repository_key(service._identity("primary")), token)[1] is OperationStatus.RECONCILED_SUCCEEDED + + +@pytest.mark.parametrize("remote_export", ["missing", "changed", "oversize"]) +def test_remote_success_reconciliation_requires_each_export_bytes(tmp_path, monkeypatch, remote_export): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, status=OperationStatus.FAILED, exit_code=1)) + with pytest.raises(ServiceError): + service.capture("daily") + token, stage, record = _retained_capture(service, staging) + snapshot = "d" * 64 + manifest = (stage / record["manifest"]["name"]).read_bytes() + exported = (stage / "db.sqlite").read_bytes() + + def inspect(plan, **kwargs): + assert "--no-lock" in plan.argv + if "locks" in plan.argv: + return _run(plan) + if "snapshots" in plan.argv: + return _run(plan, stdout=json.dumps([{"id": snapshot, + "tags": [f"bbackup-capture={token}", "bbackup-job=daily"]}]).encode()) + if plan.argv[-1].endswith("db.sqlite"): + if remote_export == "missing": + return _run(plan, status=OperationStatus.FAILED, exit_code=1) + payload = (exported + b"extra" if remote_export == "oversize" + else bytes([exported[0] ^ 1]) + exported[1:]) + else: + payload = manifest + kwargs["on_stdout_chunk"](payload) + return replace(_run(plan, stdout=payload), stdout_tail=b"") + + monkeypatch.setattr(service.runner, "run", inspect) + with pytest.raises(ServiceError, match="Remote capture export"): + service.reconcile_run("primary", token, ReconciliationOutcome.SUCCEEDED, quiescent_window_confirmed=True) + assert (stage / "db.sqlite").exists() + assert service.snapshots() == [] + assert service.ledger.inspect(repository_key(service._identity("primary")), token)[1] is OperationStatus.UNCERTAIN + + +@pytest.mark.parametrize("tamper_material", [False, True]) +def test_remote_covered_reconciliation_hashes_stored_material_before_receipt(tmp_path, monkeypatch, + tamper_material): + from bbackup.capture import DurableCaptureArtifacts + + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, status=OperationStatus.FAILED, exit_code=1)) + with pytest.raises(ServiceError): + service.capture("daily") + token, stage, record = _retained_capture(service, staging) + archive = (stage / record["exports"][0]["name"]).read_bytes() + manifest_path = stage / record["manifest"]["name"] + manifest = json.loads(manifest_path.read_text()) + procedure, evidence, material = "/private/procedure", "/private/evidence", "/private/material" + stored = {procedure: b"procedure", evidence: b"evidence", material: b"reviewed material"} + coverage = {"root_id": "dbroot", "source_id": "db", "connection": "source_pg", + "database": "app_db", "cluster_id": "primary", "major_version": "17", + "data_directory": "/private/pgdata", + "data_directory_identity": {"device": 1, "inode": 2, "uid": 3, "gid": 4, "mode": 0o700}, + "recovery_file_roots": [{"root_id": name, "path": path} + for name, path in (("procedure", procedure), ("evidence", evidence), + ("material", material))], + "recovery_artifacts": [{"root_id": name, "path": path, + "sha256": hashlib.sha256(stored[path]).hexdigest()} + for name, path in (("procedure", procedure), ("evidence", evidence), + ("material", material))]} + for key in ("baseline_digest", "system_identifier_sha256", "cluster_inventory_sha256", + "global_inventory_sha256", "role_inventory_sha256", "configuration_sha256", + "connection_material_sha256", "database_inventory_sha256", "recipe_sha256"): + coverage[key] = "a" * 64 + entry = manifest["exports"][0] + entry.update(kind="postgresql", format="postgresql-custom", source_database="app_db", + coverage=coverage, export_sha256=hashlib.sha256(archive).hexdigest()) + manifest_path.write_text(json.dumps(manifest)) + snapshot = "e" * 64 + record_owner = DurableCaptureArtifacts.load( + staging, service.state, repository_key(service._identity("primary")), token) + record_owner.record["manifest"] = {"name": manifest_path.name, + **record_owner._fingerprint(manifest_path)} + record_owner._save() + payloads = {str(stage / record["exports"][0]["name"]): archive, + str(manifest_path): manifest_path.read_bytes(), **stored} + if tamper_material: + payloads[material] += b"tampered" + snapshot_paths = [str(stage / record["exports"][0]["name"]), str(manifest_path), + procedure, evidence, material] + + def inspect(plan, **kwargs): + assert "--no-lock" in plan.argv + if "locks" in plan.argv: + return _run(plan) + if "snapshots" in plan.argv: + return _run(plan, stdout=json.dumps([{"id": snapshot, + "tags": [f"bbackup-capture={token}", "bbackup-job=daily"]}]).encode()) + if "cat" in plan.argv: + return _run(plan, stdout=json.dumps({"paths": snapshot_paths}).encode()) + payload = payloads[plan.argv[-1]] + kwargs["on_stdout_chunk"](payload) + return replace(_run(plan, stdout=payload), stdout_tail=b"") + + monkeypatch.setattr(service.runner, "run", inspect) + if tamper_material: + with pytest.raises(ServiceError, match="recovery artifact differs"): + service.reconcile_run("primary", token, ReconciliationOutcome.SUCCEEDED, + quiescent_window_confirmed=True) + assert stage.exists() and service.snapshots() == [] + else: + service.reconcile_run("primary", token, ReconciliationOutcome.SUCCEEDED, + quiescent_window_confirmed=True) + assert not stage.exists() and service.snapshots()[0]["snapshot_id"] == snapshot + + +@pytest.mark.parametrize("locked_at", ["before", "after"]) +def test_remote_reconciliation_refuses_active_restic_lock(tmp_path, monkeypatch, locked_at): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, status=OperationStatus.FAILED, exit_code=1)) + with pytest.raises(ServiceError): + service.capture("daily") + token, stage, _record = _retained_capture(service, staging) + inspected = [] + + def inspect(plan, **_kwargs): + inspected.append(plan.argv) + assert "--no-lock" in plan.argv + if "locks" in plan.argv: + count = sum("locks" in argv for argv in inspected) + return _run(plan, stdout=(b"a" * 64 + b"\n") if ( + locked_at == "before" or count == 2) else b"") + assert "snapshots" in plan.argv + return _run(plan, stdout=b"[]") + + monkeypatch.setattr(service.runner, "run", inspect) + with pytest.raises(ServiceError, match="active Restic lock"): + service.reconcile_run("primary", token, ReconciliationOutcome.FAILED, + quiescent_window_confirmed=True) + assert sum("snapshots" in argv for argv in inspected) == (locked_at == "after") + assert (stage / "db.sqlite").exists() + assert service.ledger.inspect(repository_key(service._identity("primary")), token)[1] is OperationStatus.UNCERTAIN + + +def test_remote_reconciliation_requires_quiescent_window_acknowledgment(tmp_path, monkeypatch): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, status=OperationStatus.FAILED, exit_code=1)) + with pytest.raises(ServiceError): + service.capture("daily") + token, stage, _record = _retained_capture(service, staging) + monkeypatch.setattr(service.runner, "run", lambda *_args, **_kwargs: pytest.fail( + "unacknowledged reconciliation dispatched a repository command")) + with pytest.raises(ServiceError, match="quiescent repository window"): + service.reconcile_run("primary", token, ReconciliationOutcome.FAILED) + assert (stage / "db.sqlite").exists() + assert service.ledger.inspect(repository_key(service._identity("primary")), token)[1] is OperationStatus.UNCERTAIN + + +@pytest.mark.parametrize("foreign", [False, True]) +def test_remote_reconcile_recovers_only_empty_owned_inode_gap(tmp_path, monkeypatch, foreign): + from bbackup.capture import DurableCaptureArtifacts + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + token = "e" * 32 + artifact = DurableCaptureArtifacts(staging, service.state, + repository_key(service._identity("primary")), token, "daily") + artifact._save() + artifact.directory.mkdir(mode=0o700) + if foreign: + (artifact.directory / "unexpected").write_text("keep") + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, stdout=b"" if "locks" in plan.argv else b"[]")) + if foreign: + with pytest.raises(ServiceError, match="empty owned stage"): + service.reconcile_run("primary", token, ReconciliationOutcome.ABANDONED, quiescent_window_confirmed=True) + assert (artifact.directory / "unexpected").read_text() == "keep" + assert json.loads(artifact.record_path.read_text())["stage_inode"] is None + else: + result = service.reconcile_run("primary", token, ReconciliationOutcome.ABANDONED, quiescent_window_confirmed=True) + assert result["retained_exports"] is True + assert json.loads(artifact.record_path.read_text())["stage_inode"] == artifact.directory.stat().st_ino + service.discard_exports("primary", token, result["artifact_record_sha256"]) + assert not artifact.directory.exists() + + +def test_legacy_remote_capture_reconciles_without_record_but_marked_new_refuses(tmp_path, monkeypatch): + service, _database_path, _staging = _remote_sqlite_service(tmp_path, monkeypatch) + legacy = OperationPlan.create(service._identity("primary"), "capture", ["never-run"]) + with service.ledger.begin(legacy): + pass + monkeypatch.setattr(service.runner, "run", lambda *_args, **_kwargs: pytest.fail( + "record-less legacy reconcile dispatched a command")) + result = service.reconcile_run("primary", legacy.operation_id, ReconciliationOutcome.ABANDONED) + assert result["legacy_capture"] is True + marked = OperationPlan.create(service._identity("primary"), "capture", ["never-run"]) + service.ledger.mark_capture_artifacts(marked.repository_id, marked.operation_id) + with service.ledger.begin(marked): + pass + with pytest.raises(ServiceError, match="New remote capture artifact record is missing"): + service.reconcile_run("primary", marked.operation_id, ReconciliationOutcome.ABANDONED, quiescent_window_confirmed=True) + assert service.ledger.inspect(marked.repository_id, marked.operation_id)[1] is OperationStatus.UNCERTAIN + + +def test_remote_success_cleans_only_recorded_stage(tmp_path, monkeypatch): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + snapshot = "c" * 64 + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, stdout=(json.dumps({"message_type": "summary", "snapshot_id": snapshot}) + "\n").encode())) + response = service.capture("daily") + record = json.loads((service.state / "capture-artifacts" / + f"{response['operation_id']}.json").read_text()) + assert record["phase"] == "cleaned" and record["exports"][0]["sha256"] + assert {path.name for path in staging.iterdir()} == {".bbackup-staging.lock"} + assert service.snapshots()[0]["snapshot_id"] == snapshot + from bbackup.capture import DurableCaptureArtifacts + token = response["operation_id"] + artifact = DurableCaptureArtifacts.load(staging, service.state, + repository_key(service._identity("primary")), token) + with pytest.raises(ServiceError, match="reconciled"): + service.discard_exports("primary", token, artifact.record_sha256()) + + +@pytest.mark.parametrize("foreign", ["file", "symlink"]) +def test_remote_reconciliation_rejects_foreign_stage_entry(tmp_path, monkeypatch, foreign): + service, _database_path, staging = _remote_sqlite_service(tmp_path, monkeypatch) + monkeypatch.setattr(service.runner, "run", lambda plan, **_kwargs: _run( + plan, status=OperationStatus.FAILED, exit_code=1)) + with pytest.raises(ServiceError): + service.capture("daily") + token, stage, _record = _retained_capture(service, staging) + extra = stage / "unexpected" + if foreign == "file": + extra.write_text("foreign") + else: + extra.symlink_to(tmp_path / "outside") + monkeypatch.setattr(service.runner, "run", lambda *_args, **_kwargs: pytest.fail( + "unsafe stage reached remote inspection")) + with pytest.raises(ServiceError, match="artifact"): + service.reconcile_run("primary", token, ReconciliationOutcome.FAILED, quiescent_window_confirmed=True) + assert extra.is_symlink() if foreign == "symlink" else extra.exists() + assert (stage / "db.sqlite").exists() + + +def test_remote_primary_capture_direct_source_no_cache_and_explicit_provider_env(tmp_path, monkeypatch): + service, data, staging = _remote_service(tmp_path) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fixture-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + monkeypatch.setenv("RESTIC_REPOSITORY", "poisoned") + monkeypatch.setenv("AWS_PROFILE", "poisoned") + monkeypatch.setattr("bbackup.service.shutil.disk_usage", lambda _path: SimpleNamespace(free=1)) + + class Monitor: + def __init__(self, *_args): + self.checks = 0 + + def check(self, _upcoming=0): + self.checks += 1 + + def result(self): + return {"staging_capacity_bounded": True, "free_space_reserve_hard_guaranteed": False} + + def close(self): + pass + + monkeypatch.setattr("bbackup.service.ResourceMonitor", Monitor) + observed = {} + + def run(plan, **kwargs): + observed["argv"] = plan.argv + observed["env"] = plan.env + observed["tick"] = kwargs.get("on_tick") + kwargs["on_tick"]() + return _run(plan, stdout=(json.dumps({"message_type": "summary", "snapshot_id": "a" * 64}) + "\n").encode()) + + monkeypatch.setattr(service.runner, "run", run) + captured = service.capture("daily") + assert captured["primary_destination"] == {"repository": "primary", "kind": "remote"} + assert captured["cloud_complete"] and not captured["local_complete"] + assert captured["resource_observation"]["staging_capacity_bounded"] + assert "--no-cache" in observed["argv"] + assert observed["argv"].count("--no-cache") == 1 + assert str(data) in observed["argv"] + assert observed["env"]["TMPDIR"] == str(staging) + assert "RESTIC_REPOSITORY" not in observed["env"] + assert "AWS_PROFILE" not in observed["env"] + assert observed["env"]["AWS_ACCESS_KEY_ID"] == "fixture-key" + assert list(staging.iterdir()) == [] + + +@pytest.mark.parametrize("cache_environment", [None, "HOME", "XDG_CACHE_HOME"]) +def test_local_restic_cache_behavior_follows_child_environment(service, monkeypatch, cache_environment): + monkeypatch.delenv("HOME", raising=False) + monkeypatch.delenv("XDG_CACHE_HOME", raising=False) + monkeypatch.delenv("RESTIC_CACHE_DIR", raising=False) + if cache_environment is not None: + monkeypatch.setenv(cache_environment, "/fixture/cache-root") + observed = {} + + def run(plan, **_kwargs): + observed["argv"] = plan.argv + observed["env"] = dict(plan.env) + return _run(plan) + + monkeypatch.setattr(service.runner, "run", run) + service.check("local") + + if cache_environment is None: + assert observed["argv"][1] == "--no-cache" + assert not observed["env"].get("HOME") + assert not observed["env"].get("XDG_CACHE_HOME") + else: + assert "--no-cache" not in observed["argv"] + assert observed["env"][cache_environment] == "/fixture/cache-root" + + +def test_local_restic_explicit_cache_dir_is_preserved_without_home(service, tmp_path, monkeypatch): + monkeypatch.delenv("HOME", raising=False) + monkeypatch.delenv("XDG_CACHE_HOME", raising=False) + monkeypatch.delenv("RESTIC_CACHE_DIR", raising=False) + cache_dir = str(tmp_path / "cache") + observed = {} + + def run(plan, **_kwargs): + observed["argv"] = plan.argv + return _run(plan) + + monkeypatch.setattr(service.runner, "run", run) + service._execute("local", ("restic", "--cache-dir", cache_dir, "check"), label="check") + + assert observed["argv"] == ("restic", "--cache-dir", cache_dir, "check") + + +def test_manual_remote_primary_copy_is_refused_before_any_dispatch(tmp_path, monkeypatch): + service, _data, _staging = _remote_service(tmp_path, with_replica=True) + monkeypatch.setattr(service.runner, "run", lambda *_args, **_kwargs: pytest.fail( + "unbudgeted copy child was dispatched")) + with pytest.raises(ServiceError, match="bounded resource support"): + service.copy("primary", "copy", "a" * 64) + + +def test_remote_capture_retains_real_staging_lease_until_receipt_finalization(tmp_path, monkeypatch): + from bbackup.resource_limits import ResourceMonitor + + service, _data, staging = _remote_service(tmp_path) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fixture-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + monkeypatch.setattr("bbackup.resource_limits.os.path.ismount", lambda path: path == staging) + budget = service.config.jobs[0].resource_budget + monkeypatch.setattr("bbackup.resource_limits._volume", lambda path: ( + budget.staging_max_bytes, budget.staging_max_bytes)) + + def run(plan, **kwargs): + kwargs["on_tick"]() + return _run(plan, stdout=(json.dumps({"message_type": "summary", "snapshot_id": "b" * 64}) + "\n").encode()) + + monkeypatch.setattr(service.runner, "run", run) + captured = service.capture("daily") + assert captured["snapshot_id"] == "b" * 64 + assert any(row["status"] == "succeeded" for row in service.ledger.summary()) + # Capture completed and released the shared lease; its stable lock remains. + with ResourceMonitor(staging, service.state, budget) as next_run: + next_run.check() + + +def test_remote_primary_budget_failure_is_structured_and_uncertain_after_dispatch(tmp_path, monkeypatch): + from bbackup.resource_limits import ResourceLimitError + service, _data, _staging = _remote_service(tmp_path) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fixture-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + + class Monitor: + calls = 0 + + def __init__(self, *_args): + pass + + def check(self, _upcoming=0): + self.calls += 1 + if self.calls > 1: + raise ResourceLimitError("Observed free-space reserve fell below the configured threshold") + + def result(self): + return {"staging_capacity_bounded": True} + + def close(self): + pass + + monkeypatch.setattr("bbackup.service.ResourceMonitor", Monitor) + + def run(_plan, **kwargs): + kwargs["on_tick"]() + + monkeypatch.setattr(service.runner, "run", run) + with pytest.raises(ResourceBudgetExceeded) as failure: + service.capture("daily") + assert failure.value.result["resource_budget_exceeded"] is True + assert failure.value.result["local_complete"] is False + assert failure.value.result["cloud_complete"] is False + assert "large-source" not in str(failure.value.result) + assert any(row["status"] == "uncertain" for row in service.ledger.summary()) + + def test_replica_failure_isolated_and_later_destination_runs(service, monkeypatch): job = replace(service.config.jobs[0], replicas=("first", "second")) monkeypatch.setattr(service, "_job", lambda _name: job) @@ -176,6 +1166,84 @@ def restore(archive, binding, target_database, **kwargs): assert not list(fresh.state.glob("database-restore-*")) +@pytest.mark.parametrize("tamper", ["export", "material", None]) +@pytest.mark.parametrize("material_root", ["directory", "file"]) +def test_covered_postgresql_restore_rejects_tampered_bytes_before_mutation(service, tmp_path, + monkeypatch, tamper, material_root): + from bbackup.models import DatabaseBinding + + service_file = tmp_path / "pg_service.conf" + service_file.write_text("[local]\nhost=/var/run/postgresql\nuser=postgres\n") + service_file.chmod(0o600) + binding = DatabaseBinding("postgresql", service_file=service_file, service_name="local") + config = Configuration.parse({"schema_version": 2, + "repositories": [{"name": "local"}, {"name": "copy", "role": "replica"}], + "sources": [], "jobs": []}) + fresh = BackupService(config, HostBindings(tmp_path / "restore-state", + service.bindings.repositories, {"destination": binding})) + snapshot = "a" * 64 + manifest_path = "/private/capture/bbackup-capture.json" + archive_path = "/private/capture/database.dump" + files_root = "/private/recovery" + artifact_path = files_root + "/roles.sql" + procedure_path = files_root + "/procedure.txt" + evidence_path = files_root + "/evidence.json" + archive_bytes = b"PGDMP-reviewed-export" + material_bytes = b"reviewed recovery material" + coverage = {"root_id": "database_root", "source_id": "database", + "connection": "source_pg", "database": "app_db", "cluster_id": "primary", + "major_version": "17", "data_directory": "/private/pgdata", + "data_directory_identity": {"device": 1, "inode": 2, "uid": 3, "gid": 4, "mode": 0o700}, + "recovery_file_roots": ([{"root_id": "recovery_root", "path": files_root}] + + ([{"root_id": "material_file", "path": artifact_path}] + if material_root == "file" else [])), + "recovery_artifacts": [{"root_id": ("material_file" if material_root == "file" + and path == artifact_path else "recovery_root"), "path": path, + "sha256": hashlib.sha256(content).hexdigest()} + for path, content in ((procedure_path, b"procedure"), + (evidence_path, b"evidence"), + (artifact_path, material_bytes))]} + for key in ("baseline_digest", "system_identifier_sha256", "cluster_inventory_sha256", + "global_inventory_sha256", "role_inventory_sha256", "configuration_sha256", + "connection_material_sha256", "database_inventory_sha256", "recipe_sha256"): + coverage[key] = "b" * 64 + entry = {"source": "database", "kind": "postgresql", "format": "postgresql-custom", + "path": archive_path, "source_database": "app_db", "coverage": coverage, + "export_sha256": hashlib.sha256(archive_bytes).hexdigest(), + "database_options": {"encoding": "UTF8", "collate": "C", "ctype": "C", "provider": "c"}} + payloads = {manifest_path: json.dumps({"schema_version": 2, "exports": [entry]}).encode(), + archive_path: archive_bytes if tamper != "export" else archive_bytes + b"changed", + procedure_path: b"procedure", evidence_path: b"evidence", + artifact_path: material_bytes if tamper != "material" else material_bytes + b"changed"} + metadata = json.dumps({"paths": [manifest_path, archive_path, files_root, artifact_path]}).encode() + monkeypatch.setattr(fresh, "_engine", lambda _repository: SimpleNamespace(base_args=lambda: ["restic"])) + calls = [] + + def execute(_repository, argv, *, on_stdout=None, **_kwargs): + calls.append(argv) + content = metadata if argv[-3:-1] == ["cat", "snapshot"] else payloads[argv[-1]] + if on_stdout is not None: + on_stdout(content) + return None, SimpleNamespace(stdout_bytes=len(content), stdout_tail=b"") + return None, SimpleNamespace(stdout_bytes=len(content), stdout_tail=content) + + monkeypatch.setattr(fresh, "_execute", execute) + mutated = [] + monkeypatch.setattr("bbackup.database_restore.restore_postgresql", + lambda *_args, **_kwargs: mutated.append(True) or "recovered") + if tamper is None: + result = fresh.restore_database("local", snapshot, "database", "destination", "recovered", + trusted_archive=True) + assert result["target_database"] == "recovered" + assert mutated == [True] + else: + with pytest.raises(ServiceError, match="digest differs"): + fresh.restore_database("local", snapshot, "database", "destination", "recovered", + trusted_archive=True) + assert not mutated + assert not list(fresh.state.glob("database-restore-*")) + + def test_restore_refuses_existing_and_overlapping(service, tmp_path): for target in (tmp_path / "data", tmp_path / "data" / "restored", tmp_path / "state" / "restored"): with pytest.raises(ServiceError): @@ -258,6 +1326,75 @@ def test_receipt_upgrade_refuses_nonprivate_backup_without_migrating(service, tm ] +def test_indefinite_checkpoint_receipt_and_legacy_expiry_migration(service, tmp_path): + columns = ("repository", "repository_identity", "snapshot", "operation", + "manifest_sha256", "captured_at", "expires_at") + prior = ("local", "identity", "a" * 64, "old-checkpoint", "b" * 64, + "2026-10-01T00:00:00Z", "2026-11-01T00:00:00Z") + with service._receipts() as db: + db.execute("DROP TABLE checkpoints") + db.execute( + """CREATE TABLE checkpoints ( + repository TEXT NOT NULL, repository_identity TEXT NOT NULL, + snapshot TEXT NOT NULL, operation TEXT PRIMARY KEY, + manifest_sha256 TEXT NOT NULL, captured_at TEXT NOT NULL, + expires_at TEXT NOT NULL + )""" + ) + db.execute("INSERT INTO checkpoints VALUES (?, ?, ?, ?, ?, ?, ?)", prior) + migrated = BackupService(service.config, service.bindings) + with migrated._receipts() as db: + assert db.execute("SELECT * FROM checkpoints").fetchall() == [prior] + assert db.execute("PRAGMA table_info(checkpoints)").fetchall()[-1][3] == 0 + indefinite = ("local", "identity", "c" * 64, "indefinite-checkpoint", + "d" * 64, "2026-10-02T00:00:00Z", None) + db.execute("INSERT INTO checkpoints VALUES (?, ?, ?, ?, ?, ?, ?)", indefinite) + archive, = (service.state / "state-backups").glob("receipts-*.sqlite3.gz") + recovered = tmp_path / "checkpoint-pre-migration.sqlite3" + with gzip.GzipFile(filename=archive, mode="rb") as incoming, recovered.open("xb") as output: + shutil.copyfileobj(incoming, output) + with sqlite3.connect(recovered) as db: + assert db.execute("PRAGMA integrity_check").fetchone() == ("ok",) + assert tuple(row[1] for row in db.execute("PRAGMA table_info(checkpoints)")) == columns + assert db.execute("PRAGMA table_info(checkpoints)").fetchall()[-1][3] == 1 + assert db.execute("SELECT * FROM checkpoints").fetchall() == [prior] + again = BackupService(service.config, service.bindings) + with again._receipts() as db: + assert db.execute("SELECT * FROM checkpoints ORDER BY operation").fetchall() == [indefinite, prior] + assert len(list((service.state / "state-backups").glob("receipts-*.sqlite3.gz"))) == 1 + + +@pytest.mark.parametrize("extra", [ + "repository TEXT NOT NULL COLLATE NOCASE", + "repository TEXT NOT NULL CHECK(length(repository) > 0)", + "repository TEXT NOT NULL DEFAULT 'local'", + "repository TEXT NOT NULL UNIQUE", +]) +def test_checkpoint_expiry_migration_refuses_extra_constraints(service, extra): + ddl = f"""CREATE TABLE checkpoints ( + {extra}, repository_identity TEXT NOT NULL, + snapshot TEXT NOT NULL, operation TEXT PRIMARY KEY, + manifest_sha256 TEXT NOT NULL, captured_at TEXT NOT NULL, + expires_at TEXT NOT NULL + )""" + prior = ("local", "identity", "a" * 64, "original-operation", + "b" * 64, "2026-10-01T00:00:00Z", "2026-11-01T00:00:00Z") + with service._receipts() as db: + db.execute("DROP TABLE checkpoints") + db.execute(ddl) + db.execute("INSERT INTO checkpoints VALUES (?, ?, ?, ?, ?, ?, ?)", prior) + original_sql = db.execute( + "SELECT sql FROM sqlite_master WHERE name='checkpoints'" + ).fetchone()[0] + with pytest.raises(ServiceError, match="unsupported constraints"): + BackupService(service.config, service.bindings) + with service._receipts() as db: + assert db.execute("SELECT sql FROM sqlite_master WHERE name='checkpoints'").fetchone()[0] == original_sql + assert db.execute("SELECT * FROM checkpoints").fetchall() == [prior] + assert db.execute("PRAGMA table_info(checkpoints)").fetchall()[-1][3] == 1 + assert not (service.state / "state-backups").exists() + + def test_capture_receipt_is_committed_before_ledger_success(service, monkeypatch): snapshot_id = "a" * 64 plans = [] @@ -505,6 +1642,22 @@ def test_real_capture_copy_check_restore(service, tmp_path): assert len(service.snapshots()) == 1 +@pytest.mark.integration +@pytest.mark.skipif(shutil.which("restic") is None, reason="requires restic") +def test_real_restore_verify_without_home_or_xdg_cache(service, tmp_path, monkeypatch): + monkeypatch.delenv("HOME", raising=False) + monkeypatch.delenv("XDG_CACHE_HOME", raising=False) + + service.initialize("local") + captured = service.capture("daily") + restored = tmp_path / "restored-without-cache-home" + result = service.restore("local", captured["snapshot_id"], restored) + + assert result["files_verified"] + restored_file = restored / str(tmp_path / "data" / "hello.txt").lstrip("/") + assert restored_file.read_bytes() == b"restore me\n" + + def _retention_snapshot(snapshot_id, tags=None): return { "id": snapshot_id, diff --git a/tests/test_service_status.py b/tests/test_service_status.py index c61442a..77825c1 100644 --- a/tests/test_service_status.py +++ b/tests/test_service_status.py @@ -215,3 +215,13 @@ def test_schedules_detect_duplicate_user_and_system_timers(monkeypatch): schedule, = _schedules((job,)) assert schedule["active_timer"] == "conflicting timers" assert schedule["next_run"] is None + + +def test_remote_primary_capture_does_not_report_local_capture(tmp_path): + cloud = "s3:https://storage.example/bucket/machine" + service = _StatusService(tmp_path / "state", tmp_path / "local", cloud, ( + _summary(repository_key(cloud), "capture", "succeeded", "2026-10-20T12:00:00+00:00"), + )) + facts = operational_status(service) + assert facts["last_local_capture"] is None + assert facts["last_remote_primary_capture"] == "2026-10-20T12:00:00+00:00" diff --git a/tests/test_systemd_qualification.py b/tests/test_systemd_qualification.py index ca565fc..45e9c9f 100644 --- a/tests/test_systemd_qualification.py +++ b/tests/test_systemd_qualification.py @@ -1,10 +1,93 @@ """Exercise guest evidence predicates without starting a VM or host service.""" import ast +import json +import os +import shlex +import shutil +import struct +import subprocess +import sys +from pathlib import Path from types import SimpleNamespace import pytest -from scripts.qualify_systemd import GUEST_STAGE +from scripts.qualify_systemd import COVERED_ROOT_GUEST_STAGE, GUEST_STAGE + + +def _covered_function(name, namespace): + tree = ast.parse(COVERED_ROOT_GUEST_STAGE) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) + and node.name == name) + exec(compile(ast.Module(body=[function], type_ignores=[]), + "", "exec"), namespace) + return namespace[name] + + +def _covered_literal(name): + tree = ast.parse(COVERED_ROOT_GUEST_STAGE) + assignment = next(node for node in tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == name + for target in node.targets)) + return ast.literal_eval(assignment.value) + + +@pytest.mark.parametrize("command_available", [False, True]) +def test_covered_root_acl_setter_reports_unavailable_or_refused_acl(tmp_path, command_available): + probe = tmp_path / "probe" + probe.write_bytes(b"fixture") + acl_uid = max(1000, probe.stat().st_uid + 1) + namespace = { + "pwd": SimpleNamespace(getpwall=lambda: [SimpleNamespace(pw_uid=acl_uid)]), + "os": os, + "shutil": SimpleNamespace(which=lambda _: "/usr/bin/setfacl" if command_available else None), + "subprocess": SimpleNamespace(PIPE=-1, run=lambda *args, **kwargs: + SimpleNamespace(returncode=1)), + "ctypes": SimpleNamespace(util=SimpleNamespace(find_library=lambda _: None)), + } + assert _covered_function("set_acl_if_available", namespace)(probe) is None + + +@pytest.mark.parametrize("raw, expected", [ + (struct.pack("", "exec") +def test_covered_root_guest_stage_is_valid_python_before_any_guest_boot(): + compile(COVERED_ROOT_GUEST_STAGE, "", "exec") + + +@pytest.mark.parametrize("include_fixture", [False, True]) +def test_covered_root_unit_verification_waits_for_restored_fixture(include_fixture): + calls = [] + namespace = {"Path": Path, "JOB_NAME": "covered", "APP_UNIT": "fixture.service", + "run": lambda argv: calls.append(argv)} + _covered_function("verify_installed_units", namespace)(include_fixture=include_fixture) + assert calls == [["/usr/bin/systemd-analyze", "verify", + "/etc/systemd/system/bbackup-covered.service", + "/etc/systemd/system/bbackup-covered.timer", + *(["/etc/systemd/system/fixture.service"] if include_fixture else [])]] + + +def test_covered_root_replacement_verifies_fixture_after_reconstruction(tmp_path): + material = tmp_path / "material" + recipes = material / "recipes" + recipes.mkdir(parents=True) + (material / "policy-candidate.json").write_text("{}", encoding="utf-8") + (material / "bindings-candidate.json").write_text("{}", encoding="utf-8") + events = [] + + class FinishedVerification(Exception): + pass + + def production(*args, **kwargs): + events.append(("production", args[0])) + return None, {"data": {"files_verified": True}} + + def verify_installed_units(*, include_fixture): + events.append(("verify", include_fixture)) + raise FinishedVerification + + namespace = {"mount_inputs": lambda: None, + "mount_recovery": lambda role: None, + "install_candidate": lambda candidate: None, + "read_json": lambda path: {"source_machine_digest": "source", + "snapshot_id": "snapshot"}, + "machine_digest": lambda: "replacement", + "CONFIG_DIR": tmp_path / "config", "CONFIG": tmp_path / "config" / "config.json", + "BINDINGS": tmp_path / "config" / "bindings.json", + "MATERIAL": material, "RESTORE_TARGET": tmp_path / "restore", + "shutil": shutil, "production": production, + "verify_root_enrollment": lambda: None, + "install_production_units": lambda **kwargs: events.append(("install", kwargs)), + "run": lambda argv, **kwargs: (events.append(("run", Path(argv[1]).name)) + or SimpleNamespace(returncode=0)), + "verify_installed_units": verify_installed_units} + replacement_role = _covered_function("replacement_role", namespace) + with pytest.raises(FinishedVerification): + replacement_role("candidate.whl") + assert events == [("production", "service"), ("install", {"include_fixture": False}), + ("run", "base-reconstruction.py"), ("production", "restore"), + ("run", "fixture-reconstruction.py"), ("verify", True)] + + +def test_covered_root_guest_records_command_failure_before_raising(tmp_path): + log_path = tmp_path / "guest.log" + namespace = { + "LOG": log_path, + "os": os, + "subprocess": SimpleNamespace(PIPE=-1, STDOUT=-2, + run=lambda *args, **kwargs: SimpleNamespace(returncode=2, stdout="offline wheel rejected")), + } + namespace["log"] = _covered_function("log", namespace) + guest_run = _covered_function("run", namespace) + with pytest.raises(RuntimeError, match=r"command failed \(2\)"): + guest_run(["uv", "pip", "install"]) + recorded = log_path.read_text(encoding="utf-8") + assert "COMMAND start: uv pip install" in recorded + assert "COMMAND rc=2: uv pip install" in recorded + assert "offline wheel rejected" in recorded + + +def test_covered_root_restore_diagnostic_preserves_cli_envelope(tmp_path, capsys): + from bbackup import production_cli + + diagnostic = tmp_path / "diagnostics" + diagnostic.mkdir(mode=0o700) + shim = _covered_literal("RESTORE_DIAGNOSTIC_SITE").replace( + "/var/log/bbackup-covered-root-restore-diagnostics", str(diagnostic)) + (diagnostic / "sitecustomize.py").write_text(shim, encoding="utf-8") + env = dict(os.environ) + env["PYTHONPATH"] = str(diagnostic) + ( + os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "") + startup = subprocess.run([sys.executable, "-c", "import bbackup.cli; print('ready')"], + env=env, capture_output=True, text=True, check=False) + assert startup.returncode == 0 and startup.stdout.strip() == "ready" + assert not startup.stderr + original_emit = production_cli._emit + try: + exec(compile(shim, "", "exec"), {}) + try: + raise ValueError("synthetic restore failure") + except ValueError: + production_cli._emit("restore", error={"code": "operation_failed", + "message": "sanitized"}) + finally: + production_cli._emit = original_emit + assert json.loads(capsys.readouterr().out)["errors"][0]["message"] == "sanitized" + trace = (diagnostic / "exception.txt").read_text(encoding="utf-8") + assert "ValueError: synthetic restore failure" in trace + assert (diagnostic / "exception.txt").stat().st_mode & 0o777 == 0o600 + + +def test_covered_root_restore_dispatch_keeps_cli_arguments_and_private_hook(tmp_path): + calls = [] + + def launch(argv, **kwargs): + calls.append((argv, kwargs)) + return SimpleNamespace(returncode=0, stdout='{"success": true, "data": {}}\n') + + namespace = {"BIN": Path("/opt/bbackup/venv/bin/bbackup"), + "CONFIG": Path("/etc/bbackup/covered-root/config.v3.json"), + "BINDINGS": Path("/etc/bbackup/covered-root/bindings.v3.json"), + "RESTORE_DIAGNOSTICS": tmp_path / "diagnostics", + "RESTORE_DIAGNOSTIC_SITE": _covered_literal("RESTORE_DIAGNOSTIC_SITE"), + "os": os, "json": json, "log": lambda _: None, + "subprocess": SimpleNamespace(PIPE=-1, STDOUT=-2, run=launch)} + production = _covered_function("production", namespace) + production("restore", "--repository", "local") + argv, options = calls[-1] + assert argv == ["/opt/bbackup/venv/bin/bbackup", "production", "--config", + "/etc/bbackup/covered-root/config.v3.json", "--bindings", + "/etc/bbackup/covered-root/bindings.v3.json", "restore", "--repository", "local"] + assert options["env"]["PYTHONPATH"].split(os.pathsep)[0] == str(tmp_path / "diagnostics") + assert (tmp_path / "diagnostics" / "sitecustomize.py").stat().st_mode & 0o777 == 0o600 + production("coverage", "check") + assert calls[-1][1]["env"] is None + + +def test_covered_root_restic_wrapper_records_restore_failure_only(tmp_path): + tree = ast.parse(COVERED_ROOT_GUEST_STAGE) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) + and node.name == "wrapper_script") + wrapper_literal = next(node.value for node in function.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "wrapper" + for target in node.targets)) + real = tmp_path / "restic.real" + real.write_text("#!/bin/sh\ncase \" $* \" in *' restore '*) echo 'synthetic restic error' >&2; exit 7;; esac\necho normal\n", + encoding="utf-8") + real.chmod(0o700) + diagnostic = tmp_path / "diagnostics" + diagnostic.mkdir(mode=0o700) + wrapper = tmp_path / "restic" + wrapper.write_text(ast.literal_eval(wrapper_literal) + .replace("/usr/local/libexec/bbackup/restic.real", str(real)) + .replace("/var/log/bbackup-covered-root-restore-diagnostics", str(diagnostic)), + encoding="utf-8") + wrapper.chmod(0o700) + normal = subprocess.run([str(wrapper), "snapshots"], capture_output=True, text=True, check=False) + assert normal.returncode == 0 and normal.stdout.strip() == "normal" + assert not list(diagnostic.iterdir()) + failed = subprocess.run([str(wrapper), "-r", "fixture", "restore", "snapshot"], + capture_output=True, text=True, check=False) + assert failed.returncode == 7 and failed.stderr.strip() == "synthetic restic error" + assert (diagnostic / "restic.status").read_text(encoding="utf-8") == "exit_code=7\n" + assert (diagnostic / "restic.stderr").read_text(encoding="utf-8") == "synthetic restic error\n" + assert (diagnostic / "restic.stderr").stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize(("retained", "state", "substate"), [ + (False, "inactive", "dead"), + (True, "active", "exited"), +]) +def test_covered_root_unit_proof_requires_retained_invocation_when_requested( + retained, state, substate): + properties = {"Result": "success", "ExecMainStatus": "0", + "ExecMainStartTimestamp": "Wed 2026-10-21 05:00:00 UTC", + "ExecMainExitTimestamp": "Wed 2026-10-21 05:00:01 UTC", + "ActiveState": state, "SubState": substate} + output = "\n".join(f"{key}={value}" for key, value in properties.items()) + namespace = {"rootctl": lambda *args: SimpleNamespace(stdout=output)} + unit_success = _covered_function("unit_success", namespace) + assert unit_success("fixture.service", retained=retained) + assert not unit_success("fixture.service", retained=not retained) + properties["ExecMainExitTimestamp"] = "" + output = "\n".join(f"{key}={value}" for key, value in properties.items()) + assert not unit_success("fixture.service", retained=retained) + + +def test_covered_root_fixture_retains_successful_oneshot_state(): + unit = _covered_function("app_unit_text", {})() + assert "Type=oneshot\nRemainAfterExit=yes\n" in unit + + +def test_existing_reboot_scenario_remains_the_default(monkeypatch): + from scripts import qualify_systemd as module + + monkeypatch.setattr(module.sys, "argv", ["qualify_systemd.py", "--artifact", "candidate.whl", + "--wheel-sha256", "a" * 64, "--wheelhouse", "wheels", "--requirements", "requirements.txt", + "--restic", "restic", "--uv", "uv"]) + assert module._parse_args().scenario == "timer-reboot" + monkeypatch.setattr(module.sys, "argv", [*module.sys.argv, "--scenario", "covered-root"]) + assert module._parse_args().scenario == "covered-root" + + +def test_covered_root_seeds_have_distinct_cloud_init_identities(tmp_path, monkeypatch): + from scripts import qualify_systemd as module + + written = {} + + def write_iso(stage, directory, filename, label): + target = stage / filename + target.write_text(label, encoding="ascii") + written[filename] = (directory / "user-data").read_text(encoding="utf-8") + return target + + monkeypatch.setattr(module, "_write_iso", write_iso) + source = module._create_covered_root_seed(tmp_path, "a" * 64, "noble", "source") + replacement = module._create_covered_root_seed(tmp_path, "a" * 64, "noble", "replacement") + source_id = (tmp_path / "covered-root-seed-source" / "meta-data").read_text() + replacement_id = (tmp_path / "covered-root-seed-replacement" / "meta-data").read_text() + assert source.name != replacement.name + assert "instance-id: bbackup-covered-root-source-" in source_id + assert "instance-id: bbackup-covered-root-replacement-" in replacement_id + assert source_id != replacement_id + assert "ExecStart=/usr/bin/python3 /usr/local/sbin/bbackup-covered-root-qualification.py source" in written[source.name] + assert "ExecStart=/usr/bin/python3 /usr/local/sbin/bbackup-covered-root-qualification.py replacement" in written[replacement.name] + assert "StandardOutput=journal+console" in written[source.name] + assert "journalctl --no-pager -n 40 -u bbackup-covered-root-qualification.service" in written[source.name] + assert "systemctl poweroff; exit 1" in written[source.name] + + +@pytest.mark.parametrize("accelerator", ["kvm", "tcg"]) +def test_covered_root_runner_uses_sequential_fresh_overlays_and_shared_recovery( + tmp_path, monkeypatch, accelerator): + from scripts import qualify_systemd as module + + commands = [] + + def run(args, **kwargs): + commands.append(args) + if args[1:3] == ["container", "inspect"]: + return SimpleNamespace(returncode=1, stdout="") + (tmp_path / "source-serial.log").write_text("SOURCE_COMPLETE {}\n", encoding="utf-8") + (tmp_path / "replacement-serial.log").write_text("COVERED_ROOT_RESULT {}\n", encoding="utf-8") + return SimpleNamespace(returncode=0, stdout="") + + monkeypatch.setattr(module, "_run", run) + result = module._docker_covered_root_qualification( + tmp_path, tmp_path / "ubuntu.img", tmp_path / "covered-root-seed-source.iso", + tmp_path / "covered-root-seed-replacement.iso", tmp_path / "inputs.iso", 900, + accelerator=accelerator) + command = commands[-1] + shell = command[-1] + assert "SOURCE_COMPLETE" in result and "COVERED_ROOT_RESULT" in result + assert "--device=/dev/kvm:/dev/kvm:rwm" in command if accelerator == "kvm" else "--device=/dev/kvm:/dev/kvm:rwm" not in command + assert "qemu-img create -q -f qcow2 -F qcow2 -b /ubuntu.img /work/source-guest.qcow2 16G" in shell + assert "qemu-img create -q -f qcow2 -F qcow2 -b /ubuntu.img /work/replacement-guest.qcow2 16G" in shell + source_command = shell.split(" > /work/source-serial.log", 1)[0].rsplit("; ", 1)[-1] + replacement_command = shell.split(" > /work/replacement-serial.log", 1)[0].rsplit("; ", 1)[-1] + assert "source-guest.qcow2" in source_command and "replacement-guest.qcow2" not in source_command + assert "replacement-guest.qcow2" in replacement_command and "source-guest.qcow2" not in replacement_command + assert "recovery.raw" in source_command and "recovery.raw" in replacement_command + assert "covered-root-seed-source.iso" in source_command + assert "covered-root-seed-replacement.iso" in replacement_command + assert source_command.count("inputs.iso") == 1 and replacement_command.count("inputs.iso") == 1 + assert f"accel={accelerator}" in shell + assert "-serial stdio" in shell and "-display none" in shell + assert "grep -Fq 'SOURCE_COMPLETE {' /work/source-serial.log" in shell + assert shell.index("grep -Fq 'SOURCE_COMPLETE {'") < shell.index("replacement-serial.log") + + +def test_covered_root_runner_leaves_foreign_owned_artifact_modes_alone(tmp_path, monkeypatch): + from scripts import qualify_systemd as module + + apk = tmp_path / "apk-installed.txt" + apk.write_text("installed packages", encoding="utf-8") + apk.chmod(0o664) + source = tmp_path / "source-serial.log" + source.write_text("SOURCE_COMPLETE {}\n", encoding="utf-8") + source.chmod(0o664) + original_euid = os.geteuid() + monkeypatch.setattr(module.os, "geteuid", lambda: original_euid + 1) + + def run(args, **kwargs): + return SimpleNamespace(returncode=1 if args[1:3] == ["container", "inspect"] else 0, + stdout="") + + monkeypatch.setattr(module, "_run", run) + module._docker_covered_root_qualification( + tmp_path, tmp_path / "ubuntu.img", tmp_path / "covered-root-seed-source.iso", + tmp_path / "covered-root-seed-replacement.iso", tmp_path / "inputs.iso", 900) + assert apk.stat().st_mode & 0o777 == 0o664 + assert source.stat().st_mode & 0o777 == 0o664 + + +@pytest.mark.parametrize("source_complete", [False, True]) +def test_covered_root_shell_starts_replacement_only_after_source_proof( + tmp_path, monkeypatch, source_complete): + from scripts import qualify_systemd as module + + commands = [] + + def run(args, **kwargs): + commands.append(args) + return SimpleNamespace(returncode=1 if args[1:3] == ["container", "inspect"] else 0, + stdout="") + + monkeypatch.setattr(module, "_run", run) + module._docker_covered_root_qualification( + tmp_path, tmp_path / "ubuntu.img", tmp_path / "covered-root-seed-source.iso", + tmp_path / "covered-root-seed-replacement.iso", tmp_path / "inputs.iso", 900) + shell = commands[-1][-1] + source_qemu = shell.split(" > /work/source-serial.log 2>&1", 1)[0].rsplit("; ", 1)[-1] + replacement_qemu = shell.split(" > /work/replacement-serial.log 2>&1", 1)[0].rsplit("; ", 1)[-1] + source_log = tmp_path / "source-serial.log" + replacement_marker = tmp_path / "replacement-started" + script = shell[shell.index(source_qemu):] + script = script.replace(source_qemu + " > /work/source-serial.log 2>&1", + "printf '" + ("SOURCE_COMPLETE {}" if source_complete else "BOOT_ROLE source") + + "\\n' > " + shlex.quote(str(source_log)), 1) + script = script.replace(replacement_qemu + " > /work/replacement-serial.log 2>&1", + "touch " + shlex.quote(str(replacement_marker)), 1) + script = script.replace("/work/source-serial.log", shlex.quote(str(source_log))) + result = subprocess.run(["sh", "-ec", script], capture_output=True, text=True, check=False) + assert result.returncode == (0 if source_complete else 22) + assert replacement_marker.exists() is source_complete + + +@pytest.mark.parametrize("accelerator", ["kvm", "tcg"]) +def test_guest_runner_uses_only_selected_accelerator(tmp_path, monkeypatch, accelerator): + from scripts import qualify_systemd as module + + commands = [] + + def run(args, **kwargs): + commands.append(args) + return SimpleNamespace(returncode=1 if args[1:3] == ["container", "inspect"] else 0, + stdout="guest proof") + + monkeypatch.setattr(module, "_run", run) + assert module._docker_qualification(tmp_path, tmp_path / "image", tmp_path / "seed", + tmp_path / "inputs", 600, accelerator=accelerator) == "guest proof" + command = commands[-1] + assert ("--device=/dev/kvm:/dev/kvm:rwm" in command) == (accelerator == "kvm") + assert f"accel={accelerator}" in command[-1] + assert ("-cpu host" in command[-1]) == (accelerator == "kvm") + assert "-display none" in command[-1] + + @pytest.mark.parametrize(("changes", "expected"), [ ({}, True), ({"ActiveState": "active", "SubState": "running"}, False), diff --git a/uv.lock b/uv.lock index afff335..4451f5f 100644 --- a/uv.lock +++ b/uv.lock @@ -31,7 +31,7 @@ wheels = [ [[package]] name = "bbackup" -version = "2.0.3" +version = "2.0.4" source = { editable = "." } dependencies = [ { name = "b2sdk" }, @@ -187,11 +187,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.5.20" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]]