From 94f8d36b635745f7a396d7c19b835fbb9592b4e2 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 15:18:56 -0600 Subject: [PATCH 01/12] chore(release): adopt Changie and trusted publishing --- .changes/README.md | 9 + .changes/header.tpl.md | 8 + {changelog.d => .changes/unreleased}/.gitkeep | 0 .changes/unreleased/release-standard.yaml | 5 + .../sdkharness-legal-hold-delete-refusal.yaml | 5 + .changes/v2.13.4.md | 1061 +++++++++++++++++ .changie.yaml | 32 + .github/signing-maintainers | 4 + .github/workflows/cd.yml | 188 ++- .github/workflows/ci.yml | 92 +- CHANGELOG.md | 6 +- CONTRIBUTING.md | 48 +- README.release.md | 3 - RELEASE.md | 199 ++++ b2sdk/_internal/scan/policies.py | 2 +- doc/source/contributing.rst | 2 +- noxfile.py | 168 +-- pyproject.toml | 45 +- scripts/check_changelog.py | 82 ++ scripts/prepare_release.py | 52 + scripts/release-notes.awk | 10 + scripts/release_preflight.py | 75 ++ scripts/release_security.py | 75 ++ scripts/verify_release_tag.py | 122 ++ test/integration/test_download.py | 2 +- test/unit/bucket/test_bucket.py | 12 +- .../transfer/downloader/test_parallel.py | 2 +- test/unit/test_release_tooling.py | 375 ++++++ uv.lock | 32 +- 29 files changed, 2358 insertions(+), 358 deletions(-) create mode 100644 .changes/README.md create mode 100644 .changes/header.tpl.md rename {changelog.d => .changes/unreleased}/.gitkeep (100%) create mode 100644 .changes/unreleased/release-standard.yaml create mode 100644 .changes/unreleased/sdkharness-legal-hold-delete-refusal.yaml create mode 100644 .changes/v2.13.4.md create mode 100644 .changie.yaml create mode 100644 .github/signing-maintainers delete mode 100644 README.release.md create mode 100644 RELEASE.md create mode 100644 scripts/check_changelog.py create mode 100644 scripts/prepare_release.py create mode 100644 scripts/release-notes.awk create mode 100644 scripts/release_preflight.py create mode 100644 scripts/release_security.py create mode 100644 scripts/verify_release_tag.py create mode 100644 test/unit/test_release_tooling.py diff --git a/.changes/README.md b/.changes/README.md new file mode 100644 index 000000000..7e1200841 --- /dev/null +++ b/.changes/README.md @@ -0,0 +1,9 @@ +# Release-note sources + +Add a fragment with `go run github.com/miniscruff/changie@v1.24.0 new`. +Unreleased YAML fragments belong in `unreleased/`. + +`v2.13.4.md` preserves the complete changelog history through v2.13.4, imported +without changing any released section. Future batches create one file per version. +Keep this historical archive intact. Regenerate `CHANGELOG.md` with +`go run github.com/miniscruff/changie@v1.24.0 merge` after editing a batch or header. diff --git a/.changes/header.tpl.md b/.changes/header.tpl.md new file mode 100644 index 000000000..f94cef0c6 --- /dev/null +++ b/.changes/header.tpl.md @@ -0,0 +1,8 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Unreleased entries are in [`.changes/unreleased/`](.changes/unreleased/). +See [CONTRIBUTING.md](CONTRIBUTING.md#changelog) for Changie commands. diff --git a/changelog.d/.gitkeep b/.changes/unreleased/.gitkeep similarity index 100% rename from changelog.d/.gitkeep rename to .changes/unreleased/.gitkeep diff --git a/.changes/unreleased/release-standard.yaml b/.changes/unreleased/release-standard.yaml new file mode 100644 index 000000000..317be5c59 --- /dev/null +++ b/.changes/unreleased/release-standard.yaml @@ -0,0 +1,5 @@ +kind: Infrastructure +body: >- + Migrate release notes to Changie, use PyPI Trusted Publishing with attestations, + and add release security checks and a staged signed-tag verification gate. +time: 2026-11-17T00:00:00Z diff --git a/.changes/unreleased/sdkharness-legal-hold-delete-refusal.yaml b/.changes/unreleased/sdkharness-legal-hold-delete-refusal.yaml new file mode 100644 index 000000000..24f75fc73 --- /dev/null +++ b/.changes/unreleased/sdkharness-legal-hold-delete-refusal.yaml @@ -0,0 +1,5 @@ +{ + "kind": "Infrastructure", + "body": "Make the sdkharness legal-hold delete check accept B2's `401 access_denied` response while retaining older simulator refusal shapes.", + "time": "2026-11-13T19:52:04Z" +} diff --git a/.changes/v2.13.4.md b/.changes/v2.13.4.md new file mode 100644 index 000000000..ad6d74bef --- /dev/null +++ b/.changes/v2.13.4.md @@ -0,0 +1,1061 @@ +## [2.13.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.4) - 2026-11-12 + +### Fixed + +- Notification-rule methods and B2 error interpretation no longer record request arguments (including signing secrets, custom headers, and SSE-C keys) in SDK debug tracing. + + +## [2.13.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.3) - 2026-11-06 + + +### Fixed + +- Accept `reporter=None` in `Synchronizer.sync_folders` by using a silent reporter instead of failing. +- Keep scanning a local folder after a file cannot be read or accessed when no reporter is given, instead of failing on that file. +- Parallel download: hash the data that was actually written, and surface a failing writer to producers instead of blocking them or losing the error. +- Pass `expires`, `content_disposition`, `content_encoding` and `content_language` through `Bucket.copy` to the created file instead of dropping them. +- Send `customUploadTimestamp` under its B2 wire name when starting a large file, so custom upload timestamps are no longer ignored. +- Stop the replication monitor from hanging when its consumer fails: the producer no longer blocks on a full queue and always signals the end of input. +- Store the account's allowed capabilities in SQLite with a bound parameter instead of string formatting, and reset cached upload pools when `SqliteAccountInfo.clear()` is called. + +### Infrastructure + +- Updated the locked urllib3 dependency to version 2.8.0. ([#638](https://github.com/Backblaze/b2-sdk-python/issues/638)) +- Updated Starlette, AnyIO, and virtualenv in the development tooling. Setuptools remains below 81 for liccheck compatibility and requires a dev-only dismissal; filelock and tornado retain Python 3.10-compatible dev-only locks. ([#640](https://github.com/Backblaze/b2-sdk-python/issues/640)) + + +## [2.13.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.2) - 2026-10-21 + + +### Changed + +- Require certifi 2026.10.19 or newer. + +### Infrastructure + +- Make the sdkharness `upload.stall` check runnable: it meets its request-timeout need through the documented `B2HttpApiConfig(http_session_factory=...)` instead of reporting `no-client-option`. +- Require pip 26.2.2 or newer in the Nox runner, lint, build, and test environments and update GitHub Actions to uv 0.13.4 to address the emSign CA certificate security incident. + + +## [2.13.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.1) - 2026-10-04 + +### Fixed + +- Retry uploads after a B2 request-timeout response (HTTP 408). ([#605](https://github.com/Backblaze/b2-sdk-python/issues/605)) +- Fixed honoring the Retry-After header on 5xx service errors. ([#606](https://github.com/Backblaze/b2-sdk-python/issues/606)) + +### Infrastructure + +- Added a repository-owned sdkharness customer-health contract for version-matched simulator testing. ([#610](https://github.com/Backblaze/b2-sdk-python/issues/610)) +- Moved sdkharness conformance and resilience assertions into this repository so they execute against the exact b2sdk checkout. ([#612](https://github.com/Backblaze/b2-sdk-python/issues/612)) +- Document in the sdkharness README how to run one check against the standalone simulator, and list every proxy variable the leaf checks scrub. +- Make the sdkharness leaf-check unit tests ignore `__pycache__` and other non-leaf entries. +- Repository-owned conformance checks `lock.bypass_governance` and `lock.per_file_retention` accept the 401 `access_denied` that B2 answers for a delete of a governance-locked version without bypass, and still fail if the delete is allowed or refused for an unrelated reason. +- Repository-owned sdkharness conformance and resilience checks now report an authorization error, access denied, a connection error against the simulator, or an SDK `ImportError` as `FAIL` instead of `COULD-NOT-RUN`/`SKIP`, strip proxy environment variables so a stray proxy cannot change a result, and no longer refer to files that exist only in the central harness. +- Repository-owned sdkharness conformance and resilience leaf checks now refuse to run unless they are given a loopback simulator URL, and always use the fixed simulator credential, so running one directly can no longer reach staging or production with ambient `B2_*` credentials. +- The repository-owned resilience dispatcher refuses a simulator that has already served requests, instead of letting a leaf count an earlier scenario's journal entries (for example `upload.cap_exceeded_403` reporting extra `b2_upload_file` calls) and report a false SDK failure. + + +## [2.13.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.0) - 2026-09-14 + + +### Changed + +- Follow B2's switch to SSE-B2 as the default server-side encryption: an omitted encryption setting now means the server default (SSE-B2) for both bucket defaults and file writes, and an explicit `none` is rejected before the request is sent, with `WrongEncryptionModeForBucketDefault` for bucket create/update and the new `WrongEncryptionSettingForFileWrite` for uploads and copies. `RawSimulator` applies the same rules. + +### Added + +- Add `EncryptionSetting.can_be_used_for_file_write()` and the `WrongEncryptionSettingForFileWrite` exception. + +### Infrastructure + +- Add a build-time smoke test that installs the freshly built source distribution and imports the public API shims. +- Add a checked-in Read the Docs build configuration so docs builds install the required doc dependencies and system packages. +- Bump urllib3 from 2.6.3 to 2.7.0. +- Run B2 CLI unit tests and integration tests in CI against the SDK checkout. + + +## [2.12.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.12.0) - 2026-05-05 + + +### Removed + +- Drop support for Python 3.9 and PyPy 3.9. + +### Infrastructure + +- Upgrade test dependencies versions. + + +## [2.11.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.11.0) - 2026-05-01 + + +### Fixed + +- Handle null case when encoding / decoding `content_sha1` in BaseFileVersion. ([#553](https://github.com/Backblaze/b2-sdk-python/issues/553)) +- Fix `SqliteAccountInfo` to explicitly close temporary sqlite connections used during setup, add a `close()` method for cached connection. + +### Added + +- Declare official support for Python 3.14 and include it in the CI test matrix. +- Introduce a reusable validator to ensure remote B2 file names are safe to use as local paths. + +### Infrastructure + +- Bump tornado dependency from 6.5.4 to 6.5.5. + + +## [2.10.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.4) - 2026-03-03 + + +### Fixed + +- Fixed a retry bug in `upload_unbound_stream()` small-file uploads where a retryable upload error could cause a one-shot buffered stream to be reopened after it was closed, raising `ValueError: I/O operation on closed file`. + +### Infrastructure + +- Add exponential retries for bucket creation in `testing.helpers.BucketManager`. +- Fix redudant files in the sdist after migrating from `pdm` to `uv`. + + +## [2.10.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.3) - 2026-02-23 + + +### Fixed + +- Address backwards compatibility issue for sqlite account info caused by the migration of schema to a new multi-bucket format. +- Avoid http-level retries during upload requests. +- Use `stat.S_ISDIR` check for local folder children scanning instead of `Path.is_dir` to account for an api change in Python 3.14. + +### Infrastructure + +- Bump pip version. +- Migrate from pdm to uv. +- Move pytest_plugins to top-level conftest (fix for newer pytest). +- Set temporary directory as account info config dir in tests. +- Use a separate nox dependency group for reprocible installs of nox in ci / cd. + + +## [2.10.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.2) - 2025-12-16 + + +### Fixed + +- Fix type error in retrying behavior for 429 status with Retry-After header value set. + +### Infrastructure + +- Improve debugging in integration tests bucket manager. +- Redesign `b2http` unit tests utilizing sentry `responses` library for mocking out the `requests` package. +- Update CI to not use deprecated `macos-13` runner. + + +## [2.10.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.1) - 2025-10-26 + + +### Changed + +- Improve seed generation for containers' names in tests. + + +## [2.10.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.0) - 2025-08-18 + + +### Removed + +- Drop Python 3.8 support. + +### Changed + +- Improved test helpers and fixtures and moved them to the public API. + + +## [2.9.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.9.4) - 2025-06-11 + + +### Fixed + +- Fix broken `StubAccountInfo.list_bucket_names_ids()`. +- Fix bucket name mapping in `RawSimulator.authorize_account()`. +- Fix incorrect path check in `LocalFolder.make_full_path`. + + +## [2.9.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.9.3) - 2025-05-29 + + +### Fixed + +- Fix `B2HttpApiConfig` and `Services` references in exposed `v2` API. + + +## [2.9.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.9.2) - 2025-05-29 + + +### Fixed + +- Fix incorrect reliance on v3 abstractions in apiver v2. ([#540](https://github.com/Backblaze/b2-sdk-python/issues/540)) + + +## [2.9.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.9.1) - 2025-05-28 + + +### Changed + +- Adapt authorize_account flow to multi-bucket keys. +- Migrate to b2 native api v4. +- Move url_for_api func to an internal classmethod in B2Api class. +- Release apiver v3 interface. `from b2sdk.v3 import ...` is now the recommended import, but previous versions are still supported. +- Update application key classes to support multiple bucket ids. +- Update create_key flow to multi-bucket keys. + +### Infrastructure + +- Migrate integration tests to apiver v3. + + +## [2.8.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.8.1) - 2025-04-24 + + +### Fixed + +- Fix TimeoutError handling in `b2http`. + +### Doc + +- Document params in FileRetentionSetting class. ([#532](https://github.com/Backblaze/b2-sdk-python/issues/532)) + + +## [2.8.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.8.0) - 2025-01-23 + + +### Changed + +- Migrate to B2 Native API v3. + +### Fixed + +- Fix continuation for started large files with no fully finished parts. +- Perform re-authentication for empty 401 responses returned for `HEAD` requests. + +### Infrastructure + +- Remove yapf in favor of ruff. + + +## [2.7.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.7.0) - 2024-12-12 + + +### Changed + +- Make Event Notifications generally available. ([#518](https://github.com/Backblaze/b2-sdk-python/issues/518)) +- Switch a pytest hook from path to collection_path. + +### Fixed + +- Add upload token reset after upload timeout. +- Fix file/directory permission handling for Windows during the B2 sync. + +### Infrastructure + +- Fix event notification tests when introducing new keys in API outputs. + + +## [2.6.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.6.0) - 2024-10-28 + + +### Removed + +- Remove Python 3.7 support in new releases. + Under Python 3.7 `pip` will keep resolving the latest version of the package that supports active interpreter. + Python 3.8 is now the minimum supported version, [until it reaches EOL in October 2024](https://devguide.python.org/versions/). + We encourage use of the latest stable Python release. + +### Fixed + +- Fixed datetime.utcnow() deprecation warnings under Python 3.12. + +### Added + +- Declare official support for Python 3.13 in b2sdk. + Test b2sdk against Python 3.13 in CI. + +### Infrastructure + +- Upgraded to pytest 8 (#484). + + +## [2.5.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.5.1) - 2024-08-15 + + +### Fixed + +- Fix LocalFolder.all_files(..) erroring out if one of the non-excluded directories is not readable by the user running the scan. + Warning is added to ProgressReport instead as other file access errors are. + + +## [2.5.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.5.0) - 2024-07-30 + + +### Fixed + +- Fix TruncatedOutput errors when downloading files over congested network (fixes [B2_Command_Line_Tool#554](https://github.com/Backblaze/B2_Command_Line_Tool/issues/554)). +- Ensure `FileSimulator.as_download_headers` returns `dict[str, str]` mapping. + +### Added + +- Add `unhide_file` method to Bucket class. + +### Doc + +- Improve `download_file_from_url` methods type hints. + +### Infrastructure + +- Limit max CI (Github Actions) duration to 90 minutes. + + +## [2.4.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.4.1) - 2024-06-19 + + +### Fixed + +- Fix `LocalFolder` regression (introduced in 2.4.0) which caused `LocalFolder` to not list files by path lexicographical order. + This is also a fix for `synchronizer` re-uploading files on every run in some cases. ([#502](https://github.com/Backblaze/b2-sdk-python/issues/502)) + + +## [2.4.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.4.0) - 2024-06-17 + + +### Changed + +- In `b2sdk.v3` the `B2Api` will always create `cache` from `AccountInfo` object, unless `cache` is provided explicitly. + The current stable `b2sdk.v2` remains unchanged, i.e. `DummyCache` is created by default if `account_info` was provided, but not `cache`. + Documentation for `b2sdk.v2` was updated with the new recommended usage, e.g. `B2Api(info, cache=AuthInfoCache(info))`, to achieve the same behavior as `b2sdk.v3`. ([#497](https://github.com/Backblaze/b2-sdk-python/issues/497)) + +### Fixed + +- Move scan filters before a read on filesystem access attempt. This will prevent unnecessary warnings and IO operations on paths that are not relevant to the operation. ([#456](https://github.com/Backblaze/b2-sdk-python/issues/456)) +- Fix bucket caching erroring out when using `StubAccountInfo`. + +### Added + +- Add `annotated_types` dependency for type annotations that include basic value validation. +- Add `daysFromStartingToCancelingUnfinishedLargeFiles` option to `lifecycle_rules` type annotation. +- Add non-retryable `NoPaymentHistory` exception. + API returns this exception when action (e.g. bucket creation or replication rules) is not allowed due to lack of payment history. + + +## [2.3.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.3.0) - 2024-05-15 + + +### Added + +- Add `folder_to_list_can_be_a_file` parameter to `b2sdk.v2.Bucket.ls`, that if set to `True` will allow listing a file versions if path is an exact match. + This parameter won't be included in `b2sdk.v3.Bucket.ls` and unless supplied `path` ends with `/`, the possibility of path pointing to file will be considered first. + + +## [2.2.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.2.1) - 2024-05-09 + + +### Fixed + +- Fix `__str__` of `b2sdk.v2.BucketIdNotFound` to return full error message and not just missing bucket ID value. + + +## [2.2.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.2.0) - 2024-05-08 + +### Added + +- Add `has_errors_or_warnings` method to `ProgressReport` class. + +### Fixed + +- Ensure `b2sdk.v2.b2http` emits `b2sdk.v2.BucketIdNotFound` exception instead of `b2sdk._v3.BucketIdNotFound`. ([#437](https://github.com/Backblaze/b2-sdk-python/issues/437)) +- Ensure `unprintable_to_hex` and `unprintable_to_hex` return empty string (instead of `None`) if empty string was supplied as argument. +- Skip files with invalid filenames when scanning directories (for `sync`, ...) instead of raising an exception. + + +## [2.1.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.1.0) - 2024-04-15 + + +### Changed + +- Use ParallelDownloader for small files instead of SimpleDownloader to avoid blocking on I/O. + +### Fixed + +- Fix `decode_content=True` causing an error when downloading tiny and large files. +- Prevent errors due to the use of "seekable" download strategies for seekable, but not readable files. + +### Added + +- Add set&get Event Notification rules methods to Bucket API as part of Event Notifications feature Private Preview. + See https://www.backblaze.com/blog/announcing-event-notifications/ for details. + + +## [2.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.0.0) - 2024-04-02 + + +### Removed + +- Remove `tqdm` dependency. Now `tqdm` has to be explicitly installed to use `TqdmProgressListener` class. +- Remove `[doc]` extras dependency group - moved to dev dependencies. +- Remove unnecessary `packaging` package dependency. It's functionality was never explicitly exposed. + +### Changed + +- Move non-apiver packages (e.g. packages other than `b2sdk.v1`, `b2sdk.v2`, ...) to `b2sdk._internal` to further discourage use of non-public internals. + If you accidentally used non-public internals, most likely only thing you will need to do, is import from `b2sdk.v2` instead of `b2sdk`. +- Move logging setup and `UrllibWarningFilter` class from `b2sdk.__init__.py` to `b2sdk._v3` (and thus `b2sdk.v2` & `b2sdk.v1`). + This will allow us to remove/change it in new apiver releases without the need to change the major semver version. + +### Added + +- Add `SqliteAccountInfo.get_user_account_info_path` to public API. + +### Infrastructure + +- Update to [GitHub Actions using Node 20](https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/). + + +## [1.33.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.33.0) - 2024-03-15 + + +### Fixed + +- Escape control characters whenever printing object and bucket names to improve security. +- Remove unused `setuptools` from default dependency list. + +### Added + +- Added control character escaping methods. + + +## [1.32.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.32.0) - 2024-02-26 + + +### Added + +- Add `set_thread_pool_size`, `get_thread_pool_size` to *Manger classes. + +### Infrastructure + +- Fix schema graph rendering in readthedocs documentation. + + +## [1.31.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.31.0) - 2024-02-19 + + +### Fixed + +- Remove obsolete test scripts from b2sdk package: `test_upload_url_concurrency`, `b2sdk.b2http:test_http`. ([#471](https://github.com/Backblaze/b2-sdk-python/issues/471)) + +### Added + +- Allow for `min_part_size` that is greater than default `recommended_part_size` value, without having to explicitly set `recommended_part_size` value. +- Add `GET` method support to `B2Http`. +- Add `JSON` type annotation and fix type hints in `B2Http` methods. +- Add more type hints to API methods. + + +## [1.30.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.30.1) - 2024-02-02 + + +### Fixed + +- Fix package author metadata. + + +## [1.30.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.30.0) - 2024-02-02 + + +### Fixed + +- Fix escape sequence warnings present in python 3.12. ([#458](https://github.com/Backblaze/b2-sdk-python/issues/458)) +- Handle json encoded, invalid B2 error responses, preventing exceptions such as `invalid literal for int() with base 10: 'service_unavailable'`. + +### Added + +- Add support for filters to `Bucket.ls()`. + +### Infrastructure + +- Package the library using [pdm](https://pdm-project.org), use locked dependencies in CI. +- Update `ruff` linter and apply it to all files. + + +## [1.29.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.29.1) - 2024-01-23 + + +### Fixed + +- Handle non-json encoded B2 error responses, i.e. retry on 502 and 504 errors. + +### Doc + +- Add missing import in Synchronizer docs example. + + +## [1.29.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.29.0) - 2023-12-13 + + +### Changed + +- Change v3.B2Api.authorize_account signature to make `realm` optional and `"production"` by default. + +### Added + +- Progress listener instances can now change their descriptions during run. This allows for e.g.: changing description after file headers are downloaded but before the content is fetched. + +### Infrastructure + +- Add `-v` to pytest in CI. +- Run windows pypy3.9 tests on nightly builds. + + +## [1.28.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.28.0) - 2023-12-06 + + +### Changed + +- On XDG compatible OSes (Linux, BSD), the profile file is now created in `$XDG_CONFIG_HOME` (with a fallback to `~/.config/` in absence of given env. variable). +- Replace blank `assert` with exception when size values for parts upload are misaligned. + +### Fixed + +- Streaming from empty stream no longer ends with "Empty emerge parts iterator" error. + +### Infrastructure + +- Changelog entries are now validated as a part of CI pipeline. +- Disable dependabot requests for updates unrelated to security issues. +- Fixed tests failing because of changes made to `locale.normalize` in Python 3.12. + + +## [1.27.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.27.0) - 2023-11-26 + + +### Changed + +- Add dependency on `setuptools` and `packaging` as they are not shipped by cpython 3.12 and are used in production code. + +### Fixed + +- Fix closing of passed progress listeners in `Bucket.upload` and `Bucket.copy` + + +## [1.26.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.26.0) - 2023-11-20 + + +### Added + +- Add `expires`, `content_disposition`, `content_encoding`, `content_language` arguments to various `Bucket` methods ([#357](https://github.com/Backblaze/b2-sdk-python/issues/357)) + +### Infrastructure + +- Towncrier changelog generation - to avoid conflicts when simultaneously working on PRs +- Fix towncrier generated changelog to work with mindsers/changelog-reader-action + + +## [1.25.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.25.0) - 2023-11-15 + +### Added +- Add `*_PART_SIZE`, `BUCKET_NAME_*`, `STDOUT_FILEPATH` constants +- Add `points_to_fifo`, `points_to_stdout` functions + +### Changed +- Mark `TempDir` as deprecated in favor of `tempfile.TemporaryDirectory` + +### Fixed +- Fix downloading to a non-seekable file, such as /dev/stdout +- Fix ScanPoliciesManager support for compiled regexes + +### Infrastructure +- Fix readthedocs build by updating to v2 configuration schema +- Fix spellcheck erroring out on LICENSE file +- Fix snyk reporting vulnerability due to tornado package use in docs generation +- Deduplicate test_base files in test suite +- Refactor integration tests for better pytest compatibility & eager bucket cleanup + +## [1.24.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.24.1) - 2023-09-27 + +### Fixed +- Fix missing key ID for large file encrypted with SSE-C +- Fix concatenating error message when message is None + +## [1.24.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.24.0) - 2023-08-31 + +### Added +- 'bypass_governance' flag to delete_file_version + +## [1.23.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.23.0) - 2023-08-10 + +### Added +- Add `get_file_info_by_name` to the B2Api class + +### Fixed +- Require `typing_extensions` on Python 3.11 (already required on earlier versions) for better compatibility with pydantic v2 +- Fix `RawSimulator` handling of `cache_control` parameter during tests. + +## [1.22.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.22.1) - 2023-07-24 + +### Fixed +- Fix regression in dir exclusion patterns introduced in 1.22.0 + +## [1.22.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.22.0) - 2023-07-21 + +### Added +- Declare official support of Python 3.12 +- Improved `lifecycle_rules` argument type annotations + +### Deprecated +- Deprecate `file_infos` argument. Use `file_info` instead. Old argument name won't be supported in v3. + +### Changed +- `version_utils` decorators now ignore `current_version` parameter to better fit `apiver` needs + +### Fixed +- Circular symlinks no longer cause infinite loops when syncing a folder +- Fix crash on upload retry with unbound data source + +### Infrastructure +- Remove unsupported PyPy versions (3.7, 3.8) from tests matrix and add PyPy 3.9 & 3.10 instead +- Replaced `pyflakes` with `ruff` for linting +- Refactored logic for resuming large file uploads to unify code paths, correct inconsistencies, and enhance configurability (#381) +- Automatically set copyright date when generating the docs +- Use modern type hints in documentation (achieved through combination of PEP 563 & 585 and `sphinx-autodoc-typehints`) + +## [1.21.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.21.0) - 2023-04-17 + +### Added +- Add support for custom upload timestamp +- Add support for cache control header while uploading + +### Infrastructure +- Remove dependency from `arrow` +- Build Python wheels for distribution + +## [1.20.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.20.0) - 2023-03-23 + +### Added +- Add `use_cache` parameter to `B2Api.list_buckets` + +### Changed +- Connection timeout is now being set explicitly + +### Fixed +- Small files downloaded twice + +### Infrastructure +- Disable changelog verification for dependabot PRs + +## [1.19.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.19.0) - 2023-01-24 + +### Added +- Authorizing a key for a single bucket ensures that this bucket is cached +- `Bucket.ls` operation supports wildcard matching strings +- Documentation for `AbstractUploadSource` and its children +- `InvalidJsonResponse` when the received error is not a proper JSON document +- Raising `PotentialS3EndpointPassedAsRealm` when a specific misconfiguration is suspected +- Add `large_file_sha1` support +- Add support for incremental upload and sync +- Ability to stream data from an unbound source to B2 (for example stdin) + +### Fixed +- Removed information about replication being in closed beta +- Don't throw raw `OSError` exceptions when using `DownloadedFile.save_to` to a path that doesn't exist, is a directory or the user doesn't have permissions to write to + +### Infrastructure +- Additional tests for listing files/versions +- Ensured that changelog validation only happens on pull requests +- Upgraded GitHub actions checkout to v3, python-setup to v4 +- Additional tests for `IncrementalHexDigester` + +## [1.18.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.18.0) - 2022-09-20 + +### Added +- Logging performance summary of parallel download threads +- Add `max_download_streams_per_file` parameter to B2Api class and underlying structures +- Add `is_file_lock_enabled` parameter to `Bucket.update()` and related methods + +### Fixed +- Replace `ReplicationScanResult.source_has_sse_c_enabled` with `source_encryption_mode` +- Fix `B2Api.get_key()` and `RawSimulator.delete_key()` +- Fix calling `CopySizeTooBig` exception + +### Infrastructure +- Fix nox's deprecated `session.install()` calls +- Re-enable changelog validation in CI +- StatsCollector contains context managers for gathering performance statistics + +## [1.17.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.3) - 2022-07-15 + +### Fixed +- Fix `FileVersion._get_upload_headers` when encryption key is `None` + +### Infrastructure +- Fix download integration tests on non-production environments +- Add `B2_DEBUG_HTTP` env variable to enable network-level test debugging +- Disable changelog validation temporarily + +## [1.17.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.2) - 2022-06-24 + +### Fixed +- Fix a race in progress reporter +- Fix import of replication + +## [1.17.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.1) - 2022-06-23 [YANKED] + +### Fixed +- Fix importing scan module + +## [1.17.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.0) - 2022-06-23 [YANKED] + +As in version 1.16.0, the replication API may still be unstable, however +no backward-incompatible changes are planned at this point. + +### Added +- Add `included_sources` module for keeping track of included modified third-party libraries +- Add `include_existing_files` parameter to `ReplicationSetupHelper` +- Add `get_b2sdk_doc_urls` function for extraction of external documentation URLs during runtime + +### Changed +- Downloading compressed files with `Content-Encoding` header set no longer causes them to be decompressed on the fly - it's an option +- Change the per part retry limit from 5 to 20 for data transfer operations. Please note that the retry system is not considered to be a part of the public interface and is subject to be adjusted +- Do not wait more than 64 seconds between retry attempts (unless server asks for it) +- On longer failures wait an additional (random, up to 1s) amount of time to prevent client synchronization +- Flatten `ReplicationConfiguration` interface +- Reorder actions of `ReplicationSetupHelper` to avoid zombie rules + +### Fixed +- Fix: downloading compressed files and decompressing them on the fly now does not cause a TruncatedOutput error +- Fix `AccountInfo.is_master_key()` +- Fix docstring of `SqliteAccountInfo` +- Fix lifecycle rule type in the docs + +### Infrastructure +- Add 3.11.0-beta.1 to CI +- Change Sphinx major version from 5 to 6 +- Extract folder/bucket scanning into a new `scan` module +- Enable pip cache in CI + +## [1.16.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.16.0) - 2022-04-27 + +This release contains a preview of replication support. It allows for basic +usage of B2 replication feature (currently in closed beta). + +As the interface of the sdk (and the server api) may change, the replication +support shall be considered PRIVATE interface and should be used with caution. +Please consult the documentation on how to safely use the private api interface. + +Expect substantial amount of work on sdk interface: +- The interface of `ReplicationConfiguration` WILL change +- The interface of `FileVersion.replication_status` MIGHT change +- The interface of `FileVersionDownload` MIGHT change + +### Added +- Add basic replication support to `Bucket` and `FileVersion` +- Add `is_master_key()` method to `AbstractAccountInfo` +- Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES` +- Add log tracing of `interpret_b2_error` +- Add `ReplicationSetupHelper` + +### Fixed +- Fix license test on Windows +- Fix cryptic errors when running integration tests with a non-full key + +## [1.15.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.15.0) - 2022-04-12 + +### Changed +- Don't run coverage in pypy in CI +- Introduce a common thread worker pool for all downloads +- Increase http timeout to 20min (for copy using 5GB parts) +- Remove inheritance from object (leftover from python2) +- Run unit tests on all CPUs + +### Added +- Add pypy-3.8 to test matrix +- Add support for unverified checksum upload mode +- Add dedicated exception for unverified email +- Add a parameter to customize `sync_policy_manager` +- Add parameters to set the min/max part size for large file upload/copy methods +- Add CopySourceTooBig exception +- Add an option to set a custom file version class to `FileVersionFactory` +- Add an option for B2Api to turn off hash checking for downloaded files +- Add an option for B2Api to set write buffer size for `DownloadedFile.save_to` method +- Add support for multiple profile files for SqliteAccountInfo + +### Fixed +- Fix copying objects larger than 1TB +- Fix uploading objects larger than 1TB +- Fix downloading files with unverified checksum +- Fix decoding in filename and file info of `DownloadVersion` +- Fix an off-by-one bug and other bugs in the Simulator copy functionality + +### Removed +- Drop support for Python 3.5 and Python 3.6 + +## [1.14.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.14.1) - 2022-02-23 + +### Security +- Fix setting permissions for local sqlite database (thanks to Jan Schejbal for responsible disclosure!) + +## [1.14.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.14.0) - 2021-12-23 + +### Fixed +- Relax constraint on arrow to allow for versions >= 1.0.2 + +## [1.13.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.13.0) - 2021-10-24 + +### Added +- Add support for Python 3.10 + +### Changed +- Update a list with all capabilities + +### Fixed +- Fix pypy selector in CI + +## [1.12.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.12.0) - 2021-08-06 + +### Changed +- The `importlib-metadata` requirement is less strictly bound now (just >=3.3.0 for python > 3.5). +- `B2Api` `update_file_legal_hold` and `update_file_retention_setting` now return the set values + +### Added +- `BucketIdNotFound` thrown based on B2 cloud response +- `_clone` method to `FileVersion` and `DownloadVersion` +- `delete`, `update_legal_hold`, `update_retention` and `download` methods added to `FileVersion` + +### Fixed +- FileSimulator returns special file info headers properly + +### Removed +- One unused import. + +## [1.11.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.11.0) - 2021-06-24 + +### Changed +- apiver `v2` interface released. `from b2sdk.v2 import ...` is now the recommended import, + but `from b2sdk.v1 import ...` works as before + +## [1.10.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.10.0) - 2021-06-23 + +### Added +- `get_fresh_state` method added to `FileVersion` and `Bucket` + +### Changed +- `download_file_*` methods refactored to allow for inspecting DownloadVersion before downloading the whole file +- `B2Api.get_file_info` returns a `FileVersion` object in v2 +- `B2RawApi` renamed to `B2RawHTTPApi` +- `B2HTTP` tests are now common +- `B2HttpApiConfig` class introduced to provide parameters like `user_agent_append` to `B2Api` without using internal classes in v2 +- `Bucket.update` returns a `Bucket` object in v2 +- `Bucket.ls` argument `show_versions` renamed to `latest_only` in v2 +- `B2Api` application key methods refactored to operate with dataclasses instead of dicts in v2 +- `B2Api.list_keys` is a generator lazily fetching all keys in v2 +- `account_id` and `bucket_id` added to FileVersion + +### Fixed +- Fix EncryptionSetting.from_response_headers +- Fix FileVersion.size and FileVersion.mod_time_millis type ambiguity +- Old buckets (from past tests) are cleaned up before running integration tests in a single thread + +### Removed +- Remove deprecated `SyncReport` methods + +## [1.9.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.9.0) - 2021-06-07 + +### Added +- `ScanPoliciesManager` is able to filter b2 files by upload timestamp + +### Changed +- `Synchronizer.make_file_sync_actions` and `Synchronizer.make_folder_sync_actions` were made private in v2 interface +- Refactored `sync.file.*File` and `sync.file.*FileVersion` to `sync.path.*SyncPath` in v2 +- Refactored `FileVersionInfo` to `FileVersion` in v2 +- `ScanPoliciesManager` exclusion interface changed in v2 +- `B2Api` unittests for v0, v1 and v2 are now common +- `B2Api.cancel_large_file` returns a `FileIdAndName` object instead of a `FileVersion` object in v2 +- `FileVersion` has a mandatory `api` parameter in v2 +- `B2Folder` holds a handle to B2Api +- `Bucket` unit tests for v1 and v2 are now common + +### Fixed +- Fix call to incorrect internal api in `B2Api.get_download_url_for_file_name` + +## [1.8.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.8.0) - 2021-05-21 + +### Added +- Add `get_bucket_name_or_none_from_bucket_id` to `AccountInfo` and `Cache` +- Add possibility to change realm during integration tests +- Add support for "file locks": file retention, legal hold and default bucket retention + +### Fixed +- Cleanup sync errors related to directories +- Use proper error handling in `ScanPoliciesManager` +- Application key restriction message reverted to previous form +- Added missing apiver wrappers for FileVersionInfo +- Fix crash when Content-Range header is missing +- Pin dependency versions appropriately + +### Changed +- `b2sdk.v1.sync` refactored to reflect `b2sdk.sync` structure +- Make `B2Api.get_bucket_by_id` return populated bucket objects in v2 +- Add proper support of `recommended_part_size` and `absolute_minimum_part_size` in `AccountInfo` +- Refactored `minimum_part_size` to `recommended_part_size` (the value used stays the same) +- Encryption settings, types and providers are now part of the public API + +### Removed +- Remove `Bucket.copy_file` and `Bucket.start_large_file` +- Remove `FileVersionInfo.format_ls_entry` and `FileVersionInfo.format_folder_ls_entry` + +## [1.7.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.7.0) - 2021-04-22 + +### Added +- Add `__slots__` and `__eq__` to `FileVersionInfo` for memory usage optimization and ease of testing +- Add support for SSE-C server-side encryption mode +- Add support for `XDG_CONFIG_HOME` for determining the location of `SqliteAccountInfo` db file + +### Changed +- `BasicSyncEncryptionSettingsProvider` supports different settings sets for reading and writing +- Refactored AccountInfo tests to a single file using pytest + +### Fixed +- Fix clearing cache during `authorize_account` +- Fix `ChainedStream` (needed in `Bucket.create_file` etc.) +- Make tqdm-based progress reporters less jumpy and easier to read +- Fix emerger examples in docs + +## [1.6.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.6.0) - 2021-04-08 + +### Added +- Fetch S3-compatible API URL from `authorize_account` + +### Fixed +- Exclude packages inside the test package when installing +- Fix for server response change regarding SSE + +## [1.5.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.5.0) - 2021-03-25 + +### Added +- Add `dependabot.yml` +- Add support for SSE-B2 server-side encryption mode + +### Changed +- Add upper version limit for the requirements + +### Fixed +- Pin `setuptools-scm<6.0` as `>=6.0` doesn't support Python 3.5 + +## [1.4.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.4.0) - 2021-03-03 + +### Changed +- Add an ability to provide `bucket_id` filter parameter for `list_buckets` +- Add `is_same_key` method to `AccountInfo` +- Add upper version limit for arrow dependency, because of a breaking change + +### Fixed +- Fix docs autogen + +## [1.3.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.3.0) - 2021-01-13 + +### Added +- Add custom exception for `403 transaction_cap_exceeded` +- Add `get_file_info_by_id` and `get_file_info_by_name` to `Bucket` +- `FileNotPresent` and `NonExistentBucket` now subclass new exceptions `FileOrBucketNotFound` and `ResourceNotFound` + +### Changed +- Fix missing import in the synchronization example +- Use `setuptools-scm` for versioning +- Clean up CI steps + +## [1.2.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.2.0) - 2020-11-03 + +### Added +- Add support for Python 3.9 +- Support for bucket to bucket sync +- Add a possibility to append a string to the User-Agent in `B2Http` + +### Changed +- Change default fetch count for `ls` to 10000 + +### Removed +- Drop Python 2 and Python 3.4 support :tada: +- Remove `--prefix` from `ls` (it didn't really work, use `folderName` argument) + +### Fixed +- Allow to set an empty bucket info during the update +- Fix docs generation in CI + +## [1.1.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.4) - 2020-07-15 + +### Added +- Allow specifying custom realm in B2Session.authorize_account + +## [1.1.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.2) - 2020-07-06 + +### Fixed +- Fix upload part for file range on Python 2.7 + +## [1.1.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.0) - 2020-06-24 + +### Added +- Add `list_file_versions` method to buckets. +- Add server-side copy support for large files +- Add ability to synthesize objects from local and remote sources +- Add AuthInfoCache, InMemoryCache and AbstractCache to public interface +- Add ability to filter in ScanPoliciesManager based on modification time +- Add ScanPoliciesManager and SyncReport to public interface +- Add md5 checksum to FileVersionInfo +- Add more keys to dicts returned by as_dict() methods + +### Changed +- Make sync treat hidden files as deleted +- Ignore urllib3 "connection pool is full" warning + +### Removed +- Remove arrow warnings caused by https://github.com/crsmithdev/arrow/issues/612 + +### Fixed +- Fix handling of modification time of files + +## [1.0.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.2) - 2019-10-15 + +### Changed +- Remove upper version limit for arrow dependency + +## [1.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.0) - 2019-10-03 + +### Fixed +- Minor bug fix. + +## [1.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.0)-rc1 - 2019-07-09 + +### Deprecated +- Deprecate some transitional method names to v0 in preparation for v1.0.0. + +## [0.1.10](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.10) - 2019-07-09 + +### Removed +- Remove a parameter (which did nothing, really) from `b2sdk.v1.Bucket.copy_file` signature + +## [0.1.8](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.8) - 2019-06-28 + +### Added +- Add support for b2_copy_file +- Add support for `prefix` parameter on ls-like calls + +## [0.1.6](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.6) - 2019-04-24 + +### Changed +- Rename account ID for authentication to application key ID. +Account ID is still backwards compatible, only the terminology +has changed. + +### Fixed +- Fix transferer crashing on empty file download attempt + +## [0.1.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.4) - 2019-04-04 + +### Added +Initial official release of SDK as a separate package (until now it was a part of B2 CLI) diff --git a/.changie.yaml b/.changie.yaml new file mode 100644 index 000000000..cb908b39e --- /dev/null +++ b/.changie.yaml @@ -0,0 +1,32 @@ +changesDir: .changes +unreleasedDir: unreleased +headerPath: header.tpl.md +changelogPath: CHANGELOG.md +versionExt: md +versionFormat: '## [{{.VersionNoPrefix}}](https://github.com/Backblaze/b2-sdk-python/releases/tag/v{{.VersionNoPrefix}}) - {{.Time.Format "2006-01-02"}}' +kindFormat: '### {{.Kind}}' +changeFormat: '- {{.Body}}' +kinds: + - label: Removed + auto: major + - label: Changed + auto: minor + - label: Fixed + auto: patch + - label: Deprecated + auto: minor + - label: Added + auto: minor + - label: Doc + auto: patch + - label: Infrastructure + auto: patch + - label: Security + auto: patch +newlines: + afterChangelogHeader: 0 + beforeChangelogVersion: 1 + endOfVersion: 1 + afterKind: 1 + beforeKind: 1 +envPrefix: CHANGIE_ diff --git a/.github/signing-maintainers b/.github/signing-maintainers new file mode 100644 index 000000000..b365c37e0 --- /dev/null +++ b/.github/signing-maintainers @@ -0,0 +1,4 @@ +# GitHub accounts authorized for tag signing after key enrollment. +goanpeca +ffumero2003 +sophiecarreras diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 8c0c898f2..2bf836eb2 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -2,59 +2,161 @@ name: Continuous Delivery on: push: - tags: ['v*'] # push events to matching v*, i.e. v1.0, v20.15.10 + tags: ['v*'] + workflow_dispatch: + inputs: + dry_run: + description: 'Build and check release preparation without publishing anywhere' + type: boolean + required: true + default: true + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false env: - PYTHON_DEFAULT_VERSION: "3.14" - UV_VERSION: "0.13.4" + PYTHON_DEFAULT_VERSION: '3.14' + UV_VERSION: '0.13.4' jobs: - deploy: - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - B2_PYPI_PASSWORD: ${{ secrets.B2_PYPI_PASSWORD }} + release-gates: runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + actions: read + security-events: read steps: - - name: Determine if pre-release - id: prerelease_check - run: | - export IS_PRERELEASE=$([[ ${{ github.ref }} =~ [^0-9]$ ]] && echo true || echo false) - echo "prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} - uses: actions/setup-python@v6 + persist-credentials: false + - name: Require green master and a release commit on master history + env: + GH_TOKEN: ${{ github.token }} + INPUT_DRY_RUN: ${{ inputs.dry_run }} + run: python scripts/release_preflight.py + - name: Verify maintainer tag signature + if: github.event_name == 'push' + env: + ENFORCE_SIGNED_TAGS: ${{ vars.ENFORCE_SIGNED_TAGS }} + TAG: ${{ github.ref_name }} + run: python scripts/verify_release_tag.py "$TAG" + - name: Block unresolved release security findings + env: + GH_TOKEN: ${{ secrets.RELEASE_SECURITY_TOKEN || github.token }} + REPOSITORY: ${{ github.repository }} + run: python scripts/release_security.py "$REPOSITORY" + + build: + needs: release-gates + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + version: ${{ steps.notes.outputs.version }} + prerelease: ${{ steps.notes.outputs.prerelease }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} - enable-cache: true - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Install dependencies - run: uv sync --only-group nox --locked - - name: Build the distribution - id: build + - run: uv sync --only-group nox --locked + - name: Assemble a disposable release preview + if: github.event_name == 'workflow_dispatch' + run: | + go run github.com/miniscruff/changie@v1.24.0 batch patch + go run github.com/miniscruff/changie@v1.24.0 merge + - name: Validate Changie sources + run: uv run nox -vs changelog_check + - name: Extract the exact release notes + id: notes + env: + EVENT_NAME: ${{ github.event_name }} + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = workflow_dispatch ]; then + version=$(go run github.com/miniscruff/changie@v1.24.0 latest) + version=${version#v} + else + version=${TAG#v} + test ! -n "$(find .changes/unreleased -type f ! -name .gitkeep -print -quit)" + latest=$(go run github.com/miniscruff/changie@v1.24.0 latest) + test "${latest#v}" = "$version" + fi + awk -v version="$version" -f scripts/release-notes.awk CHANGELOG.md > release-notes.md + echo "version=$version" >> "$GITHUB_OUTPUT" + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo 'prerelease=false' >> "$GITHUB_OUTPUT" + else + echo 'prerelease=true' >> "$GITHUB_OUTPUT" + fi + - name: Build and smoke-test the installed distribution run: uv run nox -vs build - - name: Read the Changelog - id: read-changelog - uses: mindsers/changelog-reader-action@v2 - with: - version: ${{ steps.build.outputs.version }} - - name: Create GitHub release and upload the distribution - id: create-release - uses: softprops/action-gh-release@v2 - with: - name: ${{ steps.build.outputs.version }} - body: ${{ steps.read-changelog.outputs.changes }} - draft: ${{ env.ACTIONS_STEP_DEBUG == 'true' }} - prerelease: ${{ steps.prerelease_check.outputs.prerelease }} - files: ${{ steps.build.outputs.asset_path }} - - name: Upload the distribution to PyPI - if: ${{ env.B2_PYPI_PASSWORD != '' && steps.prerelease_check.outputs.prerelease == 'false' }} - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.B2_PYPI_PASSWORD }} + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-distributions + path: | + dist/* + release-notes.md + if-no-files-found: error + retention-days: 7 + + publish-pypi: + needs: build + if: github.event_name == 'push' && needs.build.outputs.prerelease == 'false' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: pypi + url: https://pypi.org/p/b2sdk + permissions: + contents: read + id-token: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-distributions + - name: Publish verified distributions with PEP 740 attestations + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/ + attestations: true + + github-release: + needs: [build, publish-pypi] + if: >- + always() && github.event_name == 'push' && needs.build.result == 'success' && + (needs.publish-pypi.result == 'success' || needs.build.outputs.prerelease == 'true') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-distributions + - name: Create or update the GitHub release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + TAG: ${{ github.ref_name }} + PRERELEASE: ${{ needs.build.outputs.prerelease }} + run: | + set -euo pipefail + if gh release view "$TAG" >/dev/null 2>&1; then + gh release edit "$TAG" --title "$TAG" --notes-file release-notes.md --prerelease="$PRERELEASE" + gh release upload "$TAG" ./dist/* --clobber + else + gh release create "$TAG" --verify-tag --title "$TAG" --notes-file release-notes.md --prerelease="$PRERELEASE" ./dist/* + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b58b6acf..07b4e4b35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,22 +10,22 @@ env: PYTHON_DEFAULT_VERSION: "3.14" UV_VERSION: "0.13.4" +permissions: + contents: read + jobs: lint: timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - - uses: codespell-project/actions-codespell@2391250ab05295bddd51e36a8c6295edb6343b0e - with: - ignore_words_list: datas re-use - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -33,35 +33,49 @@ jobs: run: uv sync --only-group nox --locked - name: Run linters run: uv run nox -vs lint - - name: Validate new changelog entries - if: (contains(github.event.pull_request.labels.*.name, '-changelog') == false) && (github.event.pull_request.base.ref != '') + - name: Check spelling + # Check fragment sources; preserve immutable history and the generated changelog. + # caf is intentional escaped Unicode test data. run: | - mapfile -t changed_files < <(git diff --diff-filter=ACMR --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD) - - if [ "${#changed_files[@]}" -eq 1 ] && [ "${changed_files[0]}" = "uv.lock" ]; then - echo "Skipping changelog validation for lockfile-only change." - exit 0 - fi - - if [ -z "$(git diff --diff-filter=A --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD -- changelog.d)" ]; then - echo "No changelog item added." - exit 1 - fi - - name: Changelog validation - run: uv run nox -vs towncrier_check + uvx codespell==2.4.1 --ignore-words-list=datas,re-use,caf --skip='.git,.nox,.venv,build,dist,uv.lock,CHANGELOG.md' + uvx codespell==2.4.1 --check-hidden .changes --skip='.changes/v2.13.4.md' + changelog-check: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: ${{ env.UV_VERSION }} + - run: uv sync --only-group nox --locked + - name: Validate fragments and generated changelog + env: + EVENT_NAME: ${{ github.event_name }} + ALLOW_MISSING: ${{ contains(github.event.pull_request.labels.*.name, '-changelog') }} + run: | + args=() + if [ "$EVENT_NAME" = pull_request ]; then args+=(--base origin/master); fi + if [ "$ALLOW_MISSING" = true ]; then args+=(--allow-missing-fragment); fi + uv run nox -vs changelog_check -- "${args[@]}" build: timeout-minutes: 30 - needs: lint + needs: [lint, changelog-check] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -71,22 +85,22 @@ jobs: run: uv run nox -vs build cleanup_buckets: timeout-minutes: 30 - needs: lint + needs: [lint, changelog-check] env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead with: version: ${{ env.UV_VERSION }} @@ -121,14 +135,14 @@ jobs: extras: "full" os: "ubuntu-latest" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -160,19 +174,19 @@ jobs: - os: windows-latest python-version: "3.12" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 path: b2-sdk-python - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: repository: Backblaze/B2_Command_Line_Tool path: B2_Command_Line_Tool - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -210,14 +224,14 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 9518f6bd5..a2b61f642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the changes for the -upcoming release can be found in [changelog.d](changelog.d). - - +Unreleased entries are in [`.changes/unreleased/`](.changes/unreleased/). +See [CONTRIBUTING.md](CONTRIBUTING.md#changelog) for Changie commands. ## [2.13.5](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.5) - 2026-11-17 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c561287bc..2c18263f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,32 +26,26 @@ established by reading git tags, i.e. no code or manifest file changes are requi ## Changelog -Each PR needs to have at least one changelog (aka news) item added. This is done by creating files in `changelog.d`. -`towncrier` is used for compiling these files into [CHANGELOG.md](CHANGELOG.md). There are several types of changes -(news): - -1. fixed -2. changed -3. added -4. deprecated -5. removed -6. infrastructure -7. doc - - -The `changelog.d` file name convention is: - -1. If the PR closes a github issue: `{issue_number}.{type}.md` e.g. `157.fixed.md`. Note that the - change description still has to be complete, linking an issue is just there for convenience, a change like - `fixed #157` will not be accepted. -2. If the PR is not related to a github issue: `+{unique_string}.{type}.md` e.g. `+foobar.fixed.md`. - -These files can either be created manually, or using `towncrier` e.g. - - towncrier create -c 'write your description here' 157.fixed.md - -`towncrier create` also takes care of duplicates automatically (if there is more than 1 news fragment of one type -for a given github issue). +Add a [Changie](https://changie.dev) fragment to `.changes/unreleased/` for each +user-visible, documentation or infrastructure change. Go is required to run the +pinned tool, following the same setup as Backblaze/blazer: + +```sh +go run github.com/miniscruff/changie@v1.24.0 new +uv run --group nox nox -s changelog_check +``` + +Choose Added, Changed, Deprecated, Removed, Fixed, Doc, Infrastructure or Security. +Write a complete sentence that explains the resulting behavior. Include a public +issue link in the body when applicable; keep private findings out of fragments. +Use `--kind Fixed --body 'Correct the documented behavior.'` for noninteractive +creation. CI checks the fragment fields and reproduces the merged changelog. +A `-changelog` label can waive adding a fragment; validation still runs. Lockfile-only +updates and release PRs that batch fragments have their own checks. + +Release managers use `changie batch` and `changie merge` through the Nox release +session. See [RELEASE.md](RELEASE.md) for the complete process. The `.changes/v2.13.4.md` +archive preserves all earlier release notes; do not rewrite that history. ## Developer Info @@ -78,7 +72,7 @@ For example: $ nox -s format nox > Running session format - nox > Re-using existing virtual environment at .nox/format. + nox > Reusing existing virtual environment at .nox/format. ... $ nox --no-venv -s format diff --git a/README.release.md b/README.release.md deleted file mode 100644 index 9ebbd8303..000000000 --- a/README.release.md +++ /dev/null @@ -1,3 +0,0 @@ -# Release Process - -- Run `nox -s make_release_commit -- X.Y.Z` where `X.Y.Z` is the version you're releasing, and follow the instructions diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..46f80bca1 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,199 @@ +# Releasing b2sdk + +A release is a reviewed preparation PR merged into `master`, followed by a +maintainer's tag on that PR's merge commit. Hatch-vcs derives the package version +from Git; do not add a static version to `pyproject.toml` or `b2sdk/version.py`. +Go and Changie manage the release notes, following Backblaze/blazer's setup. + +## One-time setup + +- Install Git, GitHub CLI, Go, uv and Python 3.14. Signed GPG tags also need GnuPG; + SSH-signed tags need Git 2.34+ and OpenSSH. Authenticate `gh` with this repository. +- Configure a PyPI **Trusted Publisher** for project `b2sdk`, owner `Backblaze`, + repository `b2-sdk-python`, workflow filename `cd.yml`, environment `pypi`. + A PyPI project owner must do this at + . Protect the GitHub + `pypi` environment with maintainer approval and tag deployment rules. OIDC replaces + the old publishing credential; remove that credential after the migration is verified. +- The publish job has `id-token: write` and explicitly enables PEP 740 attestations. + It downloads built distributions and does not build code or install build dependencies. +- Ensure the release gate can read Code scanning alerts, Dependabot alerts and + Secret scanning alerts. If `GITHUB_TOKEN` cannot read all three, configure + `RELEASE_SECURITY_TOKEN` with those read permissions (prefer an installation + token managed by the organization). An unreadable or malformed source **holds** + the release; disabled scanners are not silently treated as clean. +- Select and review the intended maintainer accounts in `.github/signing-maintainers`: + `goanpeca`, `ffumero2003`, `sophiecarreras`. They are GitHub usernames, not fixture keys. + +### Enroll signing keys before enforcement + +There is no enrolled-key baseline yet. Each maintainer must create a personal GPG +or SSH signing key, register its public key with their GitHub account and configure +Git to sign tags. Review fingerprints with the maintainer through an independent +channel. The workflow resolves public keys from `https://github.com/.gpg` +and `.keys` at check time into a temporary keyring/allowed-signers file, and runs +`git verify-tag`. Do not commit a private key or test key to the maintainer list. +An SSH signing key must be present in the public `.keys` response used by the verifier. + +`ENFORCE_SIGNED_TAGS` is a repository variable. Unset or `false` means **warn-only**: +an unsigned, unknown-key or unverifiable tag is reported but does not hold the +release. Once every intended release signer is enrolled and a signed-tag check has +been verified, a repository administrator enables enforcement: + +```sh +gh variable set ENFORCE_SIGNED_TAGS --repo Backblaze/b2-sdk-python --body true +``` + +With enforcement enabled, signature failures and public-key lookup failures hold +the release. GitHub account key enrollment is the trust source; there are no +committed fingerprints. Key rotation and account security therefore require review. + +## Security gate and manual review + +Run the same gate used by delivery before planning a release: + +```sh +uv run --group nox python scripts/release_security.py Backblaze/b2-sdk-python +``` + +It holds for medium/high/critical CodeQL security alerts, medium/high/critical +Dependabot alerts with an available fix, and every open secret-scanning alert +(that API has no severity field). Low alerts and dependencies without a patched +version do not hold this gate. The output contains source counts only. Resolve or +have an authorized owner dismiss findings in their source system with a reason; +the gate has no finding waiver flag. An API error is `UNVERIFIED` and exits nonzero. + +The public workflow cannot query the organization's private Snyk tracking mirror. +Before approving the release, a maintainer must manually review unresolved +`severity:`-labelled findings in `Backblaze/b2-sdk-python_private`, following the +Confluence page **Security considerations for Backblaze Labs projects**. Coordinate +any medium/high/critical findings with the security owner. Keep private finding +details out of public issues, PRs, release notes and workflow logs. + +## Plan and prepare the release PR + +1. Assign intended public work to a release milestone, review the merged changes + since the last tag, and agree the semantic version. Breaking changes require a + major, compatible features a minor, fixes/tooling a patch; choose deliberately + when the release has mixed kinds. Changie's `auto` is a preview, not approval. +2. Start from clean, current `master`; this is a hard preparation check: + + ```sh + git fetch origin --tags + git switch master + git pull --ff-only origin master + git status --short + gh run list --repo Backblaze/b2-sdk-python --branch master --workflow ci.yml --limit 1 + git log --oneline vLAST..origin/master + uv run --group nox python scripts/release_security.py Backblaze/b2-sdk-python + ``` + + Require successful CI for the actual current master commit. A pending, absent + or failed run is a hold; do not release while main is red. +3. Review `.changes/unreleased/` and prepare an explicit version: + + ```sh + go run github.com/miniscruff/changie@v1.24.0 batch patch --dry-run + uv run --locked --group nox nox -vs make_release_commit -- X.Y.Z + git switch -c release/vX.Y.Z + ``` + + The Nox command requires a clean master worktree, including untracked files; + rejects an existing tag; runs `changie batch vX.Y.Z` then `changie merge`; and + stops. It never commits, tags or publishes. It creates `.changes/vX.Y.Z.md`, + consumes the unreleased fragments and updates `CHANGELOG.md`. Historical notes + remain in `.changes/v2.13.4.md`; do not rewrite that archive. Review the new + `## [X.Y.Z]` section and regenerate with `changie merge` after editing its batch. +4. Run local verification with no production credentials in the environment: + + ```sh + uv run --locked --group nox nox -vs lint unit build doc changelog_check + ``` + + `unit` covers v0–v3 on the configured Python interpreters. CI additionally + exercises the OS/Python matrix, CLI compatibility and credentialed integration + tests when secrets are available. A local skip is not evidence for a live test. +5. Run the local customer health check against a fresh loopback simulator. From + the central harness checkout (see `.sdkharness/README.md` for manual simulator + startup and the fixed test credentials): + + ```sh + B2SIM_SOURCE=pinned sdkharness/bin/run-health-checks.sh --target simulator --versions head b2-sdk-python + ``` + + This requires the harness to have a clean SDK clone at the release candidate. + Verify the printed revision; a PASS for another checkout does not verify this + candidate. The repository-owned health executable is `.sdkharness/tests/health-golden-path`. +6. Review the diff, commit and open the release PR: + + ```sh + git add CHANGELOG.md .changes + git commit -m 'chore(release): X.Y.Z' + git push -u origin release/vX.Y.Z + gh pr create --base master --title 'chore(release): X.Y.Z' --assignee @me + ``` + + Include the changelog, exact commit scope and verification evidence; request a + human reviewer. If the scope is tooling-only, say so. Wait for approval and + green PR CI. The maintainer merges; release preparation does not authorize a tag. + +## Non-publishing workflow dry run + +After the dispatch-enabled `cd.yml` is on the default branch: + +```sh +gh workflow run cd.yml --repo Backblaze/b2-sdk-python --ref release/vX.Y.Z -f dry_run=true +``` + +The dry run checks current master CI and the live security sources, assembles a +disposable Changie batch, extracts its notes, builds an untagged development +distribution, smoke-tests the installed v0–v3 modules and uploads CI artifacts. +Neither the PyPI publisher nor `gh release create/edit/upload` runs on dispatch; +`dry_run=false` is rejected. No tag is created. This does not prove PyPI's OIDC +configuration or attestation upload; those are only exercised by an actual approved +publish. TestPyPI is not configured by this workflow and dispatch does not publish there. +GitHub may reject dispatch before the workflow trigger exists on master; record +that as a verification limit rather than bypassing the gate. + +## Tag and publish (maintainer only) + +After the preparation PR merges, fetch its exact merge commit and verify its CI. +Tag that commit, which must be reachable from current `origin/master`: + +```sh +git fetch origin --tags +git switch master +git pull --ff-only origin master +sha=$(gh pr view RELEASE_PR --repo Backblaze/b2-sdk-python --json mergeCommit --jq .mergeCommit.oid) +git log -1 --oneline "$sha" +git merge-base --is-ancestor "$sha" origin/master +git tag -s vX.Y.Z "$sha" -m 'vX.Y.Z' +uv run --group nox python scripts/verify_release_tag.py vX.Y.Z +git push origin refs/tags/vX.Y.Z +``` + +Configure `gpg.format=ssh` and `user.signingkey` for SSH, or the GPG signing key for +GPG, before `git tag -s`. The tag workflow requires successful master and tagged +commit CI, master ancestry, the configured signature gate, a clean security gate, +matching Changie version/notes and no remaining fragments before publishing. +Stable tags publish to PyPI through OIDC with PEP 740 attestations. Prerelease tags +create a GitHub prerelease without a PyPI upload, matching the prior stable-only +publication policy. The publishing job requires the protected `pypi` environment. + +After successful PyPI upload, the workflow creates the GitHub release with the +exact changelog section. Re-running the GitHub-release job edits existing notes +and replaces assets via `gh release upload --clobber`. There are no long-lived +PyPI credentials in the workflow. Confirm the release, package version, provenance +and a fresh install before declaring the release complete. + +## Recovery + +- Before a tag is pushed: fix the PR, repeat checks and obtain approval. +- A security, CI, ancestry, signature or unreadable-source hold requires resolving + the underlying condition and re-running checks; never disable a gate to ship. +- After a tag is public: do not move or delete it. Use a new version for a new build. +- If publication fails, inspect registry state and attestations before retrying. + PyPI duplicates fail rather than silently accepting a different artifact. If + PyPI already has the correct files but GitHub release creation failed, complete + the GitHub release with the verified artifacts and extracted notes; do not + rebuild and overwrite a published package. Record the recovery in the release PR. diff --git a/b2sdk/_internal/scan/policies.py b/b2sdk/_internal/scan/policies.py index 41575346a..d42d2d7e0 100644 --- a/b2sdk/_internal/scan/policies.py +++ b/b2sdk/_internal/scan/policies.py @@ -28,7 +28,7 @@ class RegexSet: def __init__(self, regex_iterable): """ - :param regex_iterable: an interable which yields regexes + :param regex_iterable: an iterable which yields regexes """ self._compiled_list = [re.compile(r) for r in regex_iterable] diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 03c33fe9a..34f8eb175 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -49,7 +49,7 @@ For example:: $ nox -s format nox > Running session format - nox > Re-using existing virtual environment at .nox/format. + nox > Reusing existing virtual environment at .nox/format. ... $ nox --no-venv -s format diff --git a/noxfile.py b/noxfile.py index 418ef765f..4978ed20c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -12,13 +12,9 @@ import os import pathlib import platform -import re -import subprocess import nox -UPSTREAM_REPO_URL = 'git@github.com:Backblaze/b2-sdk-python.git' - CI = os.environ.get('CI') is not None NOX_PYTHONS = os.environ.get('NOX_PYTHONS') _NOX_EXTRAS = os.environ.get('NOX_EXTRAS') @@ -56,7 +52,7 @@ def _detect_python_nox_id() -> str: PYTHON_VERSIONS[-1], ) -PY_PATHS = ['b2sdk', 'test', 'noxfile.py'] +PY_PATHS = ['b2sdk', 'test', 'scripts', 'noxfile.py'] nox.options.default_venv_backend = 'uv' nox.options.reuse_existing_virtualenvs = True @@ -210,7 +206,7 @@ def build(session): "'v2': pathlib.Path(v2.__file__).resolve(), " "'v3': pathlib.Path(v3.__file__).resolve()}; " 'print(module_files); ' - "assert all(not path.is_relative_to(source_root) for path in module_files.values()), " + 'assert all(not path.is_relative_to(source_root) for path in module_files.values()), ' "f'Imported modules from checkout: {module_files!r}'; " "assert all('site-packages' in path.parts for path in module_files.values()), " "f'Imported modules from an unexpected location: {module_files!r}'" @@ -261,158 +257,14 @@ def doc_cover(session): @nox.session(python=PYTHON_DEFAULT_VERSION) def make_release_commit(session): - """ - Runs `towncrier build`, commits changes, tags, all that is left to do is pushing - """ - if session.posargs: - version = session.posargs[0] - else: - session.error('Provide -- {release_version} (X.Y.Z - without leading "v")') - - if not re.match(r'^\d+\.\d+\.\d+$', version): - session.error( - f'Provided version="{version}". Version must be of the form X.Y.Z where ' - f'X, Y and Z are integers' - ) - - local_changes = subprocess.check_output(['git', 'diff', '--stat']) - if local_changes: - session.error('Uncommitted changes detected') - - current_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode() - if current_branch != 'master': - session.log('WARNING: releasing from a branch different than master') - - uv_install(session, groups=('release',)) - session.run('towncrier', 'build', '--yes', '--version', version) - - session.log( - f'CHANGELOG updated, changes ready to commit and push\n' - f' git remote add upstream {UPSTREAM_REPO_URL!r} 2>/dev/null || git remote get-url upstream\n' - f' git commit -m "release {version}"\n' - f' git push upstream {current_branch}\n' - f'Wait for a CI workflow to complete successfully, before triggering CD by pushing a tag.\n' - f' git tag v{version}\n' - f' git push upstream v{version}\n' - f'Wait for a CD workflow to complete successfully, indicates the release is done.' - ) - - -def load_allowed_change_types( - project_toml: pathlib.Path = pathlib.Path('./pyproject.toml'), -) -> set[str]: - """ - Load the list of allowed change types from the pyproject.toml file. - """ - import tomllib - - configuration = tomllib.loads(project_toml.read_text()) - return set(entry['directory'] for entry in configuration['tool']['towncrier']['type']) - - -def is_changelog_filename_valid(filename: str, allowed_change_types: set[str]) -> tuple[bool, str]: - """ - Validates whether the given filename matches our rules. - Provides information about why it doesn't match them. - """ - error_reasons = [] - - wanted_extension = 'md' - try: - description, change_type, extension = filename.rsplit('.', maxsplit=2) - except ValueError: - # Not enough values to unpack. - return False, 'Doesn\'t follow the "..md" pattern.' - - # Check whether the filename ends with .md. - if extension != wanted_extension: - error_reasons.append(f"Doesn't end with {wanted_extension} extension.") - - # Check whether the change type is valid. - if change_type not in allowed_change_types: - error_reasons.append( - f"Change type '{change_type}' doesn't match allowed types: {allowed_change_types}." - ) - - # Check whether the description makes sense. - try: - int(description) - except ValueError: - if description[0] != '+': - error_reasons.append("Doesn't start with a number nor a plus sign.") - - return len(error_reasons) == 0, ' / '.join(error_reasons) if error_reasons else '' - - -def is_changelog_entry_valid(file_content: str) -> tuple[bool, str]: - """ - We expect the changelog entry to be a valid sentence in the English language. - This includes, but not limits to, providing a capital letter at the start - and the full-stop character at the end. - - Note: to do this "properly", tools like `nltk` and `spacy` should be used. - """ - error_reasons = [] - - # Check whether the first character is a capital letter. - # Not allowing special characters nor numbers at the very start. - if not file_content[0].isalpha() or not file_content[0].isupper(): - error_reasons.append('The first character is not a capital letter.') - - # Check if the last character is a full-stop character. - if file_content.strip()[-1] != '.': - error_reasons.append('The last character is not a full-stop character.') - - return len(error_reasons) == 0, ' / '.join(error_reasons) if error_reasons else '' + """Prepare a Changie batch on clean master; never commit, tag or publish.""" + if len(session.posargs) != 1: + session.error('Provide -- X.Y.Z (without leading v)') + session.run('python', 'scripts/prepare_release.py', session.posargs[0]) @nox.session(python=PYTHON_DEFAULT_VERSION) -def towncrier_check(session): - """ - Check whether all the entries in the changelog.d follow the expected naming convention - as well as some basic rules as to their format. - """ - expected_non_md_files = {'.gitkeep'} - allowed_change_types = load_allowed_change_types() - - is_error = False - - for filename in pathlib.Path('./changelog.d/').glob('*'): - # If that's an expected file, it's all right. - if filename.name in expected_non_md_files: - continue - - # Check whether the file matches the expected pattern. - is_valid, error_message = is_changelog_filename_valid(filename.name, allowed_change_types) - if not is_valid: - session.log(f"File {filename.name} doesn't match the expected pattern: {error_message}") - is_error = True - continue - - # Check whether the file isn't too big. - if filename.lstat().st_size > 16 * 1024: - session.log( - f'File {filename.name} content is too big – it should be smaller than 16kB.' - ) - is_error = True - continue - - # Check whether the file can be loaded as UTF-8 file. - try: - file_content = filename.read_text(encoding='utf-8') - except UnicodeDecodeError: - session.log(f'File {filename.name} is not a valid UTF-8 file.') - is_error = True - continue - - # Check whether the content of the file is anyhow valid. - is_valid, error_message = is_changelog_entry_valid(file_content) - if not is_valid: - session.log(f'File {filename.name} is not a valid changelog entry: {error_message}') - is_error = True - continue - - if is_error: - session.error( - 'Found errors in the changelog.d directory. Check logs above for more information' - ) +def changelog_check(session): + """Validate Changie fragments, generated history and optional PR coverage.""" + uv_install(session, groups=('release',)) + session.run('python', 'scripts/check_changelog.py', *session.posargs) diff --git a/pyproject.toml b/pyproject.toml index 9e80c3e3e..99337d228 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ test = [ "responses>=0.26.0,<1", ] release = [ - "towncrier==23.11.0", + "pyyaml>=6.0.3,<7", ] doc = [ "sadisplay>=0.4.9", @@ -161,46 +161,3 @@ quote-style = "single" "b2sdk/v*/**" = ["I", "F403", "F405"] "b2sdk/_v*/**" = ["I", "F403", "F405"] "test/**" = ["D", "F403", "F405"] - -[tool.towncrier] -directory = "changelog.d" -filename = "CHANGELOG.md" -start_string = "\n" -underlines = ["", "", ""] -title_format = "## [{version}](https://github.com/Backblaze/b2-sdk-python/releases/tag/v{version}) - {project_date}" -issue_format = "[#{issue}](https://github.com/Backblaze/b2-sdk-python/issues/{issue})" - -[[tool.towncrier.type]] -directory = "removed" -name = "Removed" -showcontent = true - -[[tool.towncrier.type]] -directory = "changed" -name = "Changed" -showcontent = true - -[[tool.towncrier.type]] -directory = "fixed" -name = "Fixed" -showcontent = true - -[[tool.towncrier.type]] -directory = "deprecated" -name = "Deprecated" -showcontent = true - -[[tool.towncrier.type]] -directory = "added" -name = "Added" -showcontent = true - -[[tool.towncrier.type]] -directory = "doc" -name = "Doc" -showcontent = true - -[[tool.towncrier.type]] -directory = "infrastructure" -name = "Infrastructure" -showcontent = true diff --git a/scripts/check_changelog.py b/scripts/check_changelog.py new file mode 100644 index 000000000..168dd1a8a --- /dev/null +++ b/scripts/check_changelog.py @@ -0,0 +1,82 @@ +###################################################################### +# +# File: scripts/check_changelog.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Validate fragments and require a reproducible Changie merge without modifying files.""" + +from __future__ import annotations + +import argparse +import datetime +import subprocess +from pathlib import Path + +CHANGIE = ['go', 'run', 'github.com/miniscruff/changie@v1.24.0'] +KINDS = {'Added', 'Changed', 'Deprecated', 'Removed', 'Fixed', 'Doc', 'Infrastructure', 'Security'} + + +def validate_fragment(value): + if not isinstance(value, dict) or value.get('kind') not in KINDS: + raise ValueError('A fragment needs a configured kind') + body = value.get('body') + if not isinstance(body, str) or not body.strip(): + raise ValueError('A fragment needs a nonempty body') + time = value.get('time') + if not isinstance(time, (datetime.datetime, str)): + raise ValueError('A fragment needs a timestamp') + datetime.datetime.fromisoformat(str(time).replace('Z', '+00:00')) + + +def main(): + import yaml + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base') + parser.add_argument('--allow-missing-fragment', action='store_true') + args = parser.parse_args() + root = Path.cwd() + for path in (root / '.changes/unreleased').iterdir(): + if path.name == '.gitkeep': + continue + if path.is_symlink() or not path.is_file() or path.suffix not in {'.yaml', '.yml'}: + raise ValueError('Unreleased entries must be regular YAML files') + if path.stat().st_size > 16 * 1024: + raise ValueError('Fragment exceeds 16 KiB') + validate_fragment(yaml.safe_load(path.read_text())) + merged = subprocess.run( + [*CHANGIE, 'merge', '--dry-run'], check=True, capture_output=True, text=True + ) + if merged.stdout != (root / 'CHANGELOG.md').read_text(): + raise ValueError('CHANGELOG.md differs from changie merge; regenerate it') + subprocess.run([*CHANGIE, 'batch', 'patch', '--dry-run'], check=True, capture_output=True) + if args.base and not args.allow_missing_fragment: + diff = subprocess.run( + ['git', 'diff', '--name-status', f'{args.base}...HEAD'], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + added = [line.split('\t')[-1] for line in diff if line.startswith('A\t')] + paths = {line.split('\t')[-1] for line in diff} + fragment = any( + path.startswith('.changes/unreleased/') and path.endswith(('.yaml', '.yml')) + for path in added + ) + release = 'CHANGELOG.md' in paths and any( + Path(path).parent == Path('.changes') + and path.startswith('.changes/v') + and path.endswith('.md') + for path in added + ) + if not fragment and not release and paths != {'uv.lock'}: + raise ValueError('Add a Changie fragment, or batch a release with its changelog') + print('Changelog checks passed') + + +if __name__ == '__main__': + main() diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py new file mode 100644 index 000000000..d1d344021 --- /dev/null +++ b/scripts/prepare_release.py @@ -0,0 +1,52 @@ +###################################################################### +# +# File: scripts/prepare_release.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Prepare release notes on a clean master; never commit, tag, or publish.""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +CHANGIE = ['go', 'run', 'github.com/miniscruff/changie@v1.24.0'] + + +def prepare(version, root, run=subprocess.run): + if not re.fullmatch(r'\d+\.\d+\.\d+', version): + raise ValueError('Version must be X.Y.Z without a leading v') + + def git(*args): + return run( + ['git', *args], cwd=root, check=True, capture_output=True, text=True + ).stdout.strip() + + if git('branch', '--show-current') != 'master': + raise RuntimeError('Release preparation requires master') + if git('status', '--porcelain'): + raise RuntimeError( + 'Release preparation requires a clean worktree, including untracked files' + ) + if git('tag', '--list', f'v{version}'): + raise RuntimeError('That release tag already exists') + run([*CHANGIE, 'batch', f'v{version}'], cwd=root, check=True) + run([*CHANGIE, 'merge'], cwd=root, check=True) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + sys.exit('Usage: python scripts/prepare_release.py X.Y.Z') + try: + prepare(sys.argv[1], Path.cwd()) + except (ValueError, RuntimeError, subprocess.CalledProcessError) as error: + sys.exit(str(error)) + print( + 'Release notes prepared. Create a release branch and PR; follow RELEASE.md after approval.' + ) diff --git a/scripts/release-notes.awk b/scripts/release-notes.awk new file mode 100644 index 000000000..1aecbb0d6 --- /dev/null +++ b/scripts/release-notes.awk @@ -0,0 +1,10 @@ +# Extract one exact version. Treat changelog text as data. +/^## \[/ { + if (active) exit + heading = $0 + sub(/^## \[/, "", heading) + sub(/\].*$/, "", heading) + if (heading == version) { active = 1; found = 1; next } +} +active { print } +END { if (!found) exit 1 } diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py new file mode 100644 index 000000000..4bccd95d0 --- /dev/null +++ b/scripts/release_preflight.py @@ -0,0 +1,75 @@ +###################################################################### +# +# File: scripts/release_preflight.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Require successful master CI and, for a real release, a tag on master history.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys + + +def require_green_ci(repo, sha): + result = subprocess.run( + ['gh', 'api', f'repos/{repo}/actions/workflows/ci.yml/runs?head_sha={sha}&per_page=100'], + check=True, + capture_output=True, + text=True, + ) + runs = [ + run + for run in json.loads(result.stdout)['workflow_runs'] + if run['head_sha'] == sha and run['head_branch'] == 'master' and run['event'] == 'push' + ] + if not runs or runs[0]['status'] != 'completed' or runs[0]['conclusion'] != 'success': + raise RuntimeError( + 'The latest master CI run for this commit must have completed successfully' + ) + + +def main(): + repo = os.environ['GITHUB_REPOSITORY'] + if repo != 'Backblaze/b2-sdk-python': + raise RuntimeError('Release workflow is restricted to Backblaze/b2-sdk-python') + subprocess.run(['git', 'fetch', '--no-tags', 'origin', 'master'], check=True) + + def git(*args): + return subprocess.run( + ['git', *args], check=True, capture_output=True, text=True + ).stdout.strip() + + master = git('rev-parse', 'refs/remotes/origin/master') + require_green_ci(repo, master) + if os.environ['GITHUB_EVENT_NAME'] == 'workflow_dispatch': + if os.environ.get('INPUT_DRY_RUN') != 'true': + raise RuntimeError('Manual dispatch supports dry_run=true only; it never publishes') + print('Non-publishing dry run; master CI is green.') + return + if os.environ.get('GITHUB_REF_TYPE') != 'tag': + raise RuntimeError('A release requires a tag push') + tag = os.environ['GITHUB_REF_NAME'] + if not re.fullmatch(r'v\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?', tag): + raise RuntimeError('Invalid release tag') + head = git('rev-parse', 'HEAD') + if git('rev-parse', f'refs/tags/{tag}^{{commit}}') != head: + raise RuntimeError('Checked-out revision differs from the release tag') + subprocess.run(['git', 'merge-base', '--is-ancestor', head, master], check=True) + require_green_ci(repo, head) + print('Release tag is on master history and both release and master CI are green.') + + +if __name__ == '__main__': + try: + main() + except (KeyError, ValueError, RuntimeError, subprocess.SubprocessError) as error: + # Do not echo API payloads or credential-bearing stderr. + sys.exit(f'Release preflight failed ({type(error).__name__})') diff --git a/scripts/release_security.py b/scripts/release_security.py new file mode 100644 index 000000000..21b0c9e06 --- /dev/null +++ b/scripts/release_security.py @@ -0,0 +1,75 @@ +###################################################################### +# +# File: scripts/release_security.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Fail closed on the public GitHub security sources; print counts, never alert content.""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys + +BLOCKING = {'medium', 'high', 'critical'} + + +def github_api(endpoint): + result = subprocess.run( + ['gh', 'api', '--paginate', '--slurp', endpoint], + check=True, + capture_output=True, + text=True, + ) + pages = json.loads(result.stdout) + if not isinstance(pages, list) or not all(isinstance(page, list) for page in pages): + raise ValueError('Invalid paginated response') + return [item for page in pages for item in page] + + +def check(repo, api=github_api): + if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo): + raise ValueError('Expected owner/repository') + holds = [] + for source in ('dependabot', 'code-scanning', 'secret-scanning'): + try: + items = api(f'repos/{repo}/{source}/alerts?state=open&per_page=100') + if not isinstance(items, list) or not all(isinstance(item, dict) for item in items): + raise ValueError('Invalid alert response') + count = 0 + for item in items: + if source == 'dependabot': + severity = item['security_advisory']['severity'] + if severity not in BLOCKING | {'low'}: + raise ValueError('Unknown security severity') + count += severity in BLOCKING and bool( + item['security_vulnerability']['first_patched_version'] + ) + elif source == 'code-scanning': + rule = item['rule'] + severity = rule.get('security_severity_level') + if severity not in BLOCKING | {'low', None}: + raise ValueError('Unknown security severity') + if severity is None and 'security' in rule.get('tags', []): + raise ValueError('Security alert has no severity') + count += severity in BLOCKING + else: + # The secret-scanning API has no severity. Every open credential exposure holds. + count += 1 + print(f'{source}: {len(items)} open; {count} blocking') + if count: + holds.append(f'{source}: {count} blocking alert(s)') + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, RuntimeError): + holds.append(f'{source}: UNVERIFIED; cannot read or classify the required source') + return holds + + +if __name__ == '__main__': + for hold in (holds := check(sys.argv[1] if len(sys.argv) == 2 else 'Backblaze/b2-sdk-python')): + print(f'HOLD: {hold}') + sys.exit(bool(holds)) diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py new file mode 100644 index 000000000..e0f0b381e --- /dev/null +++ b/scripts/verify_release_tag.py @@ -0,0 +1,122 @@ +###################################################################### +# +# File: scripts/verify_release_tag.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Verify tags using the enrolled maintainers' GitHub public GPG and SSH keys.""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +import tempfile +import urllib.error +import urllib.request +from pathlib import Path + + +def enforcing(value): + normalized = value.strip().lower() + if normalized not in {'', 'false', 'off', '0', 'true', 'on', '1'}: + raise ValueError('ENFORCE_SIGNED_TAGS must be true or false') + return normalized in {'true', 'on', '1'} + + +def fetch_keys(url): + try: + with urllib.request.urlopen(url, timeout=15) as response: + content = response.read(1024 * 1024 + 1) + if len(content) > 1024 * 1024: + raise ValueError('Public key response exceeds size limit') + return content.decode('utf-8') + except urllib.error.HTTPError as error: + if error.code == 404: + return '' + raise + + +def verify(tag, root, *, enforce, maintainers, fetch=fetch_keys): + verified = False + try: + if not re.fullmatch(r'v\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?', tag): + raise ValueError('Invalid release tag') + if not maintainers or any(not re.fullmatch(r'[A-Za-z0-9-]+', name) for name in maintainers): + raise ValueError('Invalid maintainer list') + with tempfile.TemporaryDirectory(prefix='b2sdk-signers-') as temporary: + home = Path(temporary) + home.chmod(0o700) + ssh_keys = [] + for name in maintainers: + gpg_keys = fetch(f'https://github.com/{name}.gpg') + if gpg_keys and shutil.which('gpg'): + subprocess.run( + ['gpg', '--batch', '--homedir', str(home), '--import'], + input=gpg_keys, + text=True, + capture_output=True, + check=True, + ) + for key in fetch(f'https://github.com/{name}.keys').splitlines(): + if key.strip(): + ssh_keys.append(f'{name} {key}') + signers = home / 'allowed_signers' + signers.write_text('\n'.join(ssh_keys) + '\n') + commands = [ + ['git', '-c', 'gpg.format=openpgp', 'verify-tag', tag], + [ + 'git', + '-c', + 'gpg.format=ssh', + '-c', + f'gpg.ssh.allowedSignersFile={signers}', + 'verify-tag', + tag, + ], + ] + for command in commands: + result = subprocess.run( + command, + cwd=root, + env={**os.environ, 'GNUPGHOME': str(home)}, + capture_output=True, + text=True, + ) + if result.returncode == 0: + verified = True + break + except (OSError, ValueError, subprocess.SubprocessError, urllib.error.URLError): + verified = False + if verified: + print('Tag signature verified against an enrolled GitHub maintainer key.') + else: + level = 'error' if enforce else 'warning' + print( + f'::{level}::Tag signature unverified; enroll maintainer keys before enabling enforcement.' + ) + return verified or not enforce + + +if __name__ == '__main__': + root = Path(__file__).resolve().parents[1] + names = [ + line.strip() + for line in (root / '.github/signing-maintainers').read_text().splitlines() + if line.strip() and not line.startswith('#') + ] + try: + result = verify( + sys.argv[1], + root, + enforce=enforcing(os.getenv('ENFORCE_SIGNED_TAGS', '')), + maintainers=names, + ) + except (IndexError, ValueError): + sys.exit('Invalid tag or ENFORCE_SIGNED_TAGS setting') + sys.exit(not result) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index 4b3ec1d11..e564f953f 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -100,7 +100,7 @@ def test_small_unverified(self): @pytest.mark.parametrize('size_multiplier', [1, 100]) def test_gzip(b2_auth_data, bucket, tmp_path, b2_api, size_multiplier, realm): - """Test downloading gzipped files of varius sizes with and without content-encoding.""" + """Test downloading gzipped files of various sizes with and without content-encoding.""" source_file = tmp_path / 'compressed_file.gz' downloaded_uncompressed_file = tmp_path / 'downloaded_uncompressed_file' downloaded_compressed_file = tmp_path / 'downloaded_compressed_file' diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 86ff96fc1..82077a5d8 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -2863,9 +2863,9 @@ def test_download_to_non_seekable_file(self): file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') non_seekable_strategies = [ - strat - for strat in self.bucket.api.services.download_manager.strategies - if not isinstance(strat, ParallelDownloader) + strategy + for strategy in self.bucket.api.services.download_manager.strategies + if not isinstance(strategy, ParallelDownloader) ] context = ( contextlib.nullcontext() @@ -2888,9 +2888,9 @@ def test_download_to_seekable_but_no_read_file(self): file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') non_seekable_strategies = [ - strat - for strat in self.bucket.api.services.download_manager.strategies - if not isinstance(strat, ParallelDownloader) + strategy + for strategy in self.bucket.api.services.download_manager.strategies + if not isinstance(strategy, ParallelDownloader) ] context = ( contextlib.nullcontext() diff --git a/test/unit/internal/transfer/downloader/test_parallel.py b/test/unit/internal/transfer/downloader/test_parallel.py index eda3d4be6..ea7ca95b2 100644 --- a/test/unit/internal/transfer/downloader/test_parallel.py +++ b/test/unit/internal/transfer/downloader/test_parallel.py @@ -158,7 +158,7 @@ def iter_content(chunk_size=1, decode_unicode=False): def test_download_file__data_stream_error__multiple_errors_recovery( apiver_module, b2api, bucket, downloader, output_file ): - """Test downloader handles multiple half-failed requests and still downlaods entire file.""" + """Test downloader handles multiple half-failed requests and still downloads entire file.""" # This works since each part is attempted up to 15 times before giving up file_size = 100 mock_response, download_version = mock_download_response_factory( diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py new file mode 100644 index 000000000..02b57af4d --- /dev/null +++ b/test/unit/test_release_tooling.py @@ -0,0 +1,375 @@ +###################################################################### +# +# File: test/unit/test_release_tooling.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations + +import json +import runpy +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[2] + + +def tooling(name): + return runpy.run_path(str(ROOT / 'scripts' / f'{name}.py')) + + +@pytest.fixture +def release_repo(tmp_path): + def git(*args): + return subprocess.run( + ['git', *args], cwd=tmp_path, check=True, capture_output=True, text=True + ) + + git('init', '--initial-branch=master') + git('config', 'user.name', 'Release test') + git('config', 'user.email', 'release-test@example.invalid') + git('commit', '--allow-empty', '-m', 'Initial fixture') + return tmp_path, git + + +def test_release_preparation_refuses_other_branches(release_repo): + root, git = release_repo + git('switch', '-c', 'feature') + with pytest.raises(RuntimeError, match='master'): + tooling('prepare_release')['prepare']('2.13.5', root) + + +def test_release_preparation_refuses_untracked_changes(release_repo): + root, _ = release_repo + (root / 'pending').write_text('pending') + with pytest.raises(RuntimeError, match='clean'): + tooling('prepare_release')['prepare']('2.13.5', root) + + +@pytest.mark.parametrize('version', ['v2.13.5', '2.13', '2.13.5;echo unsafe', '2.13.5\n']) +def test_release_preparation_rejects_invalid_versions(release_repo, version): + root, _ = release_repo + with pytest.raises(ValueError, match='X.Y.Z'): + tooling('prepare_release')['prepare'](version, root) + + +def test_release_preparation_refuses_an_existing_tag(release_repo): + root, git = release_repo + git('tag', 'v2.13.5') + with pytest.raises(RuntimeError, match='exists'): + tooling('prepare_release')['prepare']('2.13.5', root) + + +def test_release_preparation_batches_then_merges_without_tagging(release_repo): + root, git = release_repo + calls = [] + + def run(args, **kwargs): + if args[0] == 'go': + calls.append(args) + return subprocess.CompletedProcess(args, 0, '') + return subprocess.run(args, **kwargs) + + tooling('prepare_release')['prepare']('2.13.5', root, run=run) + assert [call[3:] for call in calls] == [['batch', 'v2.13.5'], ['merge']] + assert git('tag', '--list').stdout == '' + + +def alerts(dependabot=None, code=None, secrets=None): + def api(endpoint): + if '/dependabot/' in endpoint: + return dependabot or [] + if '/code-scanning/' in endpoint: + return code or [] + if '/secret-scanning/' in endpoint: + return secrets or [] + raise AssertionError('Unexpected endpoint') + + return api + + +def test_security_gate_accepts_empty_sources(): + assert tooling('release_security')['check']('owner/repo', api=alerts()) == [] + + +@pytest.mark.parametrize('severity', ['medium', 'high', 'critical']) +def test_security_gate_blocks_fixable_dependabot_alerts(severity): + source = { + 'security_advisory': {'severity': severity}, + 'security_vulnerability': {'first_patched_version': {'identifier': '9.0'}}, + } + assert tooling('release_security')['check']('owner/repo', api=alerts(dependabot=[source])) + + +def test_security_gate_does_not_block_low_or_unpatched_dependabot_alerts(): + sources = [ + { + 'security_advisory': {'severity': 'low'}, + 'security_vulnerability': {'first_patched_version': {'identifier': '9.0'}}, + }, + { + 'security_advisory': {'severity': 'high'}, + 'security_vulnerability': {'first_patched_version': None}, + }, + ] + assert tooling('release_security')['check']('owner/repo', api=alerts(dependabot=sources)) == [] + + +@pytest.mark.parametrize('severity', ['medium', 'high', 'critical']) +def test_security_gate_blocks_code_scanning_alerts(severity): + source = {'rule': {'security_severity_level': severity}} + assert tooling('release_security')['check']('owner/repo', api=alerts(code=[source])) + + +def test_security_gate_blocks_open_secret_scanning_alerts(): + assert tooling('release_security')['check']('owner/repo', api=alerts(secrets=[{}])) + + +def test_security_gate_fails_closed_when_a_source_is_unreadable(): + def denied(endpoint): + raise RuntimeError('Access denied') + + assert len(tooling('release_security')['check']('owner/repo', api=denied)) == 3 + + +def test_security_gate_fails_closed_on_malformed_response(): + assert tooling('release_security')['check']('owner/repo', api=lambda endpoint: {}) + + +@pytest.mark.parametrize( + 'value, expected', [('', False), ('false', False), ('true', True), ('on', True)] +) +def test_signed_tag_enforcement_rollout(value, expected): + assert tooling('verify_release_tag')['enforcing'](value) is expected + + +def test_signed_tag_enforcement_rejects_a_misspelled_setting(): + with pytest.raises(ValueError): + tooling('verify_release_tag')['enforcing']('tru') + + +def test_unsigned_tag_is_warning_only_until_enrolled(release_repo): + root, git = release_repo + git('tag', '-a', 'v2.13.5', '-m', 'Unsigned fixture') + verify = tooling('verify_release_tag')['verify'] + assert verify('v2.13.5', root, enforce=False, maintainers=['example'], fetch=lambda url: '') + assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=lambda url: '') + + +def test_ssh_signed_tag_requires_an_enrolled_key(release_repo, tmp_path): + root, git = release_repo + # Generated fixture material stays in pytest's temporary directory. + key = tmp_path / 'fixture-signing-key' + subprocess.run(['ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(key)], check=True) + git( + '-c', + 'gpg.format=ssh', + '-c', + f'user.signingkey={key}', + 'tag', + '-s', + 'v2.13.5', + '-m', + 'Fixture', + ) + verify = tooling('verify_release_tag')['verify'] + public_key = key.with_suffix('.pub').read_text() + + # Key files are untracked fixtures; signature verification itself does not mutate the repo. + def fetch(url): + return public_key if url.endswith('.keys') else '' + + assert verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch) + assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=lambda url: '') + + +@pytest.mark.parametrize( + 'value', [None, {}, {'kind': 'Fixed'}, {'kind': 'Unknown', 'body': 'Entry.'}] +) +def test_changie_fragment_validation_rejects_incomplete_entries(value): + with pytest.raises(ValueError): + tooling('check_changelog')['validate_fragment'](value) + + +def test_changie_fragment_validation_accepts_a_complete_entry(): + tooling('check_changelog')['validate_fragment']( + {'kind': 'Fixed', 'body': 'Correct a release check.', 'time': '2026-11-17T00:00:00Z'} + ) + + +@pytest.mark.parametrize( + 'status, conclusion, branch, event', + [ + ('completed', 'failure', 'master', 'push'), + ('in_progress', None, 'master', 'push'), + ('completed', 'success', 'feature', 'push'), + ('completed', 'success', 'master', 'pull_request'), + ], +) +def test_release_preflight_rejects_red_pending_or_unrelated_ci( + monkeypatch, status, conclusion, branch, event +): + function = tooling('release_preflight')['require_green_ci'] + response = { + 'workflow_runs': [ + { + 'head_sha': 'abc', + 'head_branch': branch, + 'event': event, + 'status': status, + 'conclusion': conclusion, + } + ] + } + monkeypatch.setattr( + subprocess, + 'run', + lambda args, **kwargs: subprocess.CompletedProcess(args, 0, json.dumps(response)), + ) + with pytest.raises(RuntimeError, match='successfully'): + function('owner/repo', 'abc') + + +def test_release_preflight_requires_the_latest_master_run_to_pass(monkeypatch): + function = tooling('release_preflight')['require_green_ci'] + response = { + 'workflow_runs': [ + { + 'head_sha': 'abc', + 'head_branch': 'master', + 'event': 'push', + 'status': 'completed', + 'conclusion': 'failure', + }, + { + 'head_sha': 'abc', + 'head_branch': 'master', + 'event': 'push', + 'status': 'completed', + 'conclusion': 'success', + }, + ] + } + monkeypatch.setattr( + subprocess, + 'run', + lambda args, **kwargs: subprocess.CompletedProcess(args, 0, json.dumps(response)), + ) + with pytest.raises(RuntimeError): + function('owner/repo', 'abc') + + +def test_release_preflight_accepts_successful_master_ci(monkeypatch): + function = tooling('release_preflight')['require_green_ci'] + response = { + 'workflow_runs': [ + { + 'head_sha': 'abc', + 'head_branch': 'master', + 'event': 'push', + 'status': 'completed', + 'conclusion': 'success', + } + ] + } + monkeypatch.setattr( + subprocess, + 'run', + lambda args, **kwargs: subprocess.CompletedProcess(args, 0, json.dumps(response)), + ) + function('owner/repo', 'abc') + + +@pytest.mark.parametrize('merged', [False, True]) +def test_release_preflight_requires_tagged_commit_on_master(release_repo, monkeypatch, merged): + root, git = release_repo + base = git('rev-parse', 'HEAD').stdout.strip() + git('update-ref', 'refs/remotes/origin/master', base) + if not merged: + git('switch', '-c', 'unmerged-release') + git('commit', '--allow-empty', '-m', 'Unmerged fixture') + git('tag', '-a', 'v2.13.5', '-m', 'Fixture') + function = tooling('release_preflight')['main'] + function.__globals__['require_green_ci'] = lambda repo, sha: None + real_run = subprocess.run + + def run(args, **kwargs): + if args[:2] == ['git', 'fetch']: + return subprocess.CompletedProcess(args, 0) + return real_run(args, **kwargs) + + monkeypatch.setattr(subprocess, 'run', run) + monkeypatch.chdir(root) + monkeypatch.setenv('GITHUB_REPOSITORY', 'Backblaze/b2-sdk-python') + monkeypatch.setenv('GITHUB_EVENT_NAME', 'push') + monkeypatch.setenv('GITHUB_REF_TYPE', 'tag') + monkeypatch.setenv('GITHUB_REF_NAME', 'v2.13.5') + if merged: + function() + else: + with pytest.raises(subprocess.CalledProcessError): + function() + + +def test_dispatch_cannot_enable_publishing(release_repo, monkeypatch): + root, git = release_repo + git('update-ref', 'refs/remotes/origin/master', git('rev-parse', 'HEAD').stdout.strip()) + function = tooling('release_preflight')['main'] + function.__globals__['require_green_ci'] = lambda repo, sha: None + real_run = subprocess.run + + def run(args, **kwargs): + if args[:2] == ['git', 'fetch']: + return subprocess.CompletedProcess(args, 0) + return real_run(args, **kwargs) + + monkeypatch.setattr(subprocess, 'run', run) + monkeypatch.chdir(root) + monkeypatch.setenv('GITHUB_REPOSITORY', 'Backblaze/b2-sdk-python') + monkeypatch.setenv('GITHUB_EVENT_NAME', 'workflow_dispatch') + monkeypatch.setenv('INPUT_DRY_RUN', 'false') + with pytest.raises(RuntimeError, match='never publishes'): + function() + + +def test_release_notes_extracts_exact_version_without_running_content(tmp_path): + changelog = tmp_path / 'CHANGELOG.md' + changelog.write_text('## [2.13.5] - today\n\n### Fixed\n\n- Entry.\n\n## [2.13.4]\n\n- Old.\n') + result = subprocess.run( + [ + 'awk', + '-v', + 'version=2.13.5', + '-f', + str(ROOT / 'scripts/release-notes.awk'), + str(changelog), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert 'Entry.' in result.stdout + assert 'Old.' not in result.stdout + + +def test_release_notes_fails_when_version_is_missing(tmp_path): + changelog = tmp_path / 'CHANGELOG.md' + changelog.write_text('## [2.13.4]\n') + result = subprocess.run( + [ + 'awk', + '-v', + 'version=2.13.5', + '-f', + str(ROOT / 'scripts/release-notes.awk'), + str(changelog), + ], + capture_output=True, + text=True, + ) + assert result.returncode != 0 diff --git a/uv.lock b/uv.lock index 46673c7fb..5bf93507b 100644 --- a/uv.lock +++ b/uv.lock @@ -103,7 +103,7 @@ nox = [ { name = "pip" }, ] release = [ - { name = "towncrier" }, + { name = "pyyaml" }, ] test = [ { name = "coverage" }, @@ -159,7 +159,7 @@ nox = [ { name = "nox", specifier = ">=2025.11.12" }, { name = "pip", specifier = ">=26.2.2" }, ] -release = [{ name = "towncrier", specifier = "==23.11.0" }] +release = [{ name = "pyyaml", specifier = ">=6.0.3,<7" }] test = [ { name = "coverage", specifier = ">=7.13.5,<8" }, { name = "pip", specifier = ">=26.2.2" }, @@ -598,19 +598,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", size = 8769, upload-time = "2022-07-01T12:21:02.467Z" }, ] -[[package]] -name = "incremental" -version = "24.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/3c/82e84109e02c492f382c711c58a3dd91badda6d746def81a1465f74dc9f5/incremental-24.11.0.tar.gz", hash = "sha256:87d3480dbb083c1d736222511a8cf380012a8176c2456d01ef483242abbbcf8c", size = 24000, upload-time = "2025-11-28T02:30:17.861Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/55/0f4df2a44053867ea9cbea73fc588b03c55605cd695cee0a3d86f0029cb2/incremental-24.11.0-py3-none-any.whl", hash = "sha256:a34450716b1c4341fe6676a0598e88a39e04189f4dce5dc96f656e040baa10b3", size = 21109, upload-time = "2025-11-28T02:30:16.442Z" }, -] - [[package]] name = "iniconfig" version = "2.3.0" @@ -1765,21 +1752,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/57/4a85c617b6349962320b4afbe1386d5b05fd57218ee8ad6e82e0110ec9ee/tornado-6.6-cp315-cp315t-win_arm64.whl", hash = "sha256:7de62e2dfdf559fd7154d2b9aa58ff3d9cff8304ae0519341feb082618bc2a33", size = 530780, upload-time = "2026-10-21T15:31:03.425Z" }, ] -[[package]] -name = "towncrier" -version = "23.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "incremental" }, - { name = "jinja2" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e0/43/bb5161b32c538a099e1e1a8b79eedd4556dca925facc3b11e0ea7ab96d40/towncrier-23.11.0.tar.gz", hash = "sha256:13937c247e3f8ae20ac44d895cf5f96a60ad46cfdcc1671759530d7837d9ee5d", size = 51483, upload-time = "2023-11-08T14:27:30.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/36/5a/6546c8ea2bee64c9e9552e06fdb31118a4f23a12aa161581b5fefd070152/towncrier-23.11.0-py3-none-any.whl", hash = "sha256:2e519ca619426d189e3c98c99558fe8be50c9ced13ea1fc20a4a353a95d2ded7", size = 49327, upload-time = "2023-11-08T14:27:28.45Z" }, -] - [[package]] name = "tqdm" version = "4.67.1" From 51c66448ed453ae7b423bd5ea7c03a2981627869 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 15:25:15 -0600 Subject: [PATCH 02/12] fix(release): isolate enrolled tag signer trust --- scripts/verify_release_tag.py | 10 +++++++++- test/unit/test_release_tooling.py | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py index e0f0b381e..945df00dd 100644 --- a/scripts/verify_release_tag.py +++ b/scripts/verify_release_tag.py @@ -69,7 +69,15 @@ def verify(tag, root, *, enforce, maintainers, fetch=fetch_keys): signers = home / 'allowed_signers' signers.write_text('\n'.join(ssh_keys) + '\n') commands = [ - ['git', '-c', 'gpg.format=openpgp', 'verify-tag', tag], + [ + 'git', + '-c', + 'gpg.format=openpgp', + '-c', + f'gpg.ssh.allowedSignersFile={signers}', + 'verify-tag', + tag, + ], [ 'git', '-c', diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py index 02b57af4d..4d876b234 100644 --- a/test/unit/test_release_tooling.py +++ b/test/unit/test_release_tooling.py @@ -33,6 +33,8 @@ def git(*args): git('init', '--initial-branch=master') git('config', 'user.name', 'Release test') git('config', 'user.email', 'release-test@example.invalid') + git('config', 'commit.gpgsign', 'false') + git('config', 'tag.gpgsign', 'false') git('commit', '--allow-empty', '-m', 'Initial fixture') return tmp_path, git @@ -187,6 +189,12 @@ def fetch(url): assert verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch) assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=lambda url: '') + # A pre-existing local/global signer configuration cannot expand enrollment. + external = root / 'external-signers' + external.write_text(f'external {public_key}') + git('config', 'gpg.ssh.allowedSignersFile', str(external)) + assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=lambda url: '') + @pytest.mark.parametrize( 'value', [None, {}, {'kind': 'Fixed'}, {'kind': 'Unknown', 'body': 'Entry.'}] From 1efe0a52808326bc6a2f3957d16d48ff651b99b9 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 15:27:22 -0600 Subject: [PATCH 03/12] fix(release): support prerelease preparation and SSH-only signers --- RELEASE.md | 4 +++ scripts/prepare_release.py | 4 +-- scripts/verify_release_tag.py | 4 ++- test/unit/test_release_tooling.py | 54 +++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 46f80bca1..31b0d329b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -104,6 +104,10 @@ details out of public issues, PRs, release notes and workflow logs. consumes the unreleased fragments and updates `CHANGELOG.md`. Historical notes remain in `.changes/v2.13.4.md`; do not rewrite that archive. Review the new `## [X.Y.Z]` section and regenerate with `changie merge` after editing its batch. + For a prerelease, pass the full SemVer version instead, for example + `uv run --locked --group nox nox -vs make_release_commit -- 2.14.0-rc.1`. + Keep that exact suffix in the branch, Changie batch and tag (`v2.14.0-rc.1`); + preparing `2.14.0` does not prepare the notes for `2.14.0-rc.1`. 4. Run local verification with no production credentials in the environment: ```sh diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index d1d344021..0390c475f 100644 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -20,8 +20,8 @@ def prepare(version, root, run=subprocess.run): - if not re.fullmatch(r'\d+\.\d+\.\d+', version): - raise ValueError('Version must be X.Y.Z without a leading v') + if not re.fullmatch(r'\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?', version): + raise ValueError('Version must be X.Y.Z or X.Y.Z-rc.1 without a leading v') def git(*args): return run( diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py index 945df00dd..1b6ac0e3e 100644 --- a/scripts/verify_release_tag.py +++ b/scripts/verify_release_tag.py @@ -56,12 +56,14 @@ def verify(tag, root, *, enforce, maintainers, fetch=fetch_keys): for name in maintainers: gpg_keys = fetch(f'https://github.com/{name}.gpg') if gpg_keys and shutil.which('gpg'): + # GitHub returns an armored stub for accounts with no GPG keys. + # An unsuccessful import must still allow an enrolled SSH key. subprocess.run( ['gpg', '--batch', '--homedir', str(home), '--import'], input=gpg_keys, text=True, capture_output=True, - check=True, + check=False, ) for key in fetch(f'https://github.com/{name}.keys').splitlines(): if key.strip(): diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py index 4d876b234..80885d5b2 100644 --- a/test/unit/test_release_tooling.py +++ b/test/unit/test_release_tooling.py @@ -11,6 +11,7 @@ import json import runpy +import shutil import subprocess from pathlib import Path @@ -82,6 +83,59 @@ def run(args, **kwargs): assert git('tag', '--list').stdout == '' +def test_release_preparation_preserves_the_prerelease_version(release_repo): + root, _ = release_repo + calls = [] + + def run(args, **kwargs): + if args[0] == 'go': + calls.append(args) + return subprocess.CompletedProcess(args, 0, '') + return subprocess.run(args, **kwargs) + + tooling('prepare_release')['prepare']('2.14.0-rc.1', root, run=run) + assert calls[0][3:] == ['batch', 'v2.14.0-rc.1'] + + +def test_github_no_gpg_keys_response_does_not_prevent_ssh_verification(release_repo, monkeypatch): + root, git = release_repo + key = root / 'fixture-key' + subprocess.run(['ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(key)], check=True) + git( + '-c', + 'gpg.format=ssh', + '-c', + f'user.signingkey={key}', + 'tag', + '-s', + 'v2.13.5', + '-m', + 'Fixture', + ) + public_key = key.with_suffix('.pub').read_text() + real_run, real_which = subprocess.run, shutil.which + + def run(args, **kwargs): + if args[0] == 'gpg': + if kwargs.get('check'): + raise subprocess.CalledProcessError(2, args) + return subprocess.CompletedProcess(args, 2) + return real_run(args, **kwargs) + + def fetch(url): + if url.endswith('.keys'): + return public_key + return ( + "-----BEGIN PGP PUBLIC KEY BLOCK-----\nNote: This user hasn't uploaded any GPG keys.\n" + ) + + monkeypatch.setattr(subprocess, 'run', run) + monkeypatch.setattr(shutil, 'which', lambda name: 'gpg' if name == 'gpg' else real_which(name)) + assert tooling('verify_release_tag')['verify']( + 'v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch + ) + + def alerts(dependabot=None, code=None, secrets=None): def api(endpoint): if '/dependabot/' in endpoint: From 56ea057ef25b988f80eba22891668b26e919fe6f Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 15:33:15 -0600 Subject: [PATCH 04/12] fix(release): share the supported prerelease grammar Disclose the direct PyYAML release-tooling dependency and retain it for fragment validation. --- RELEASE.md | 7 ++++++- scripts/prepare_release.py | 4 +++- scripts/release_preflight.py | 4 +++- scripts/release_versions.py | 14 ++++++++++++++ scripts/verify_release_tag.py | 4 +++- test/unit/test_release_tooling.py | 25 +++++++++++++++++++++---- 6 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 scripts/release_versions.py diff --git a/RELEASE.md b/RELEASE.md index 31b0d329b..b37ba103d 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -4,6 +4,8 @@ A release is a reviewed preparation PR merged into `master`, followed by a maintainer's tag on that PR's merge commit. Hatch-vcs derives the package version from Git; do not add a static version to `pyproject.toml` or `b2sdk/version.py`. Go and Changie manage the release notes, following Backblaze/blazer's setup. +The release tooling directly depends on PyYAML for fragment validation, replacing +Towncrier in the `release` dependency group. This does not add a runtime dependency. ## One-time setup @@ -104,7 +106,10 @@ details out of public issues, PRs, release notes and workflow logs. consumes the unreleased fragments and updates `CHANGELOG.md`. Historical notes remain in `.changes/v2.13.4.md`; do not rewrite that archive. Review the new `## [X.Y.Z]` section and regenerate with `changie merge` after editing its batch. - For a prerelease, pass the full SemVer version instead, for example + Supported prerelease suffixes are `-a.N`, `-b.N` and `-rc.N`, where `N` is a + nonnegative integer without leading zeroes. This is a deliberately restricted, + Python-compatible interface, not arbitrary SemVer identifiers or build metadata. + All three release entry points use the same validator. For example, run `uv run --locked --group nox nox -vs make_release_commit -- 2.14.0-rc.1`. Keep that exact suffix in the branch, Changie batch and tag (`v2.14.0-rc.1`); preparing `2.14.0` does not prepare the notes for `2.14.0-rc.1`. diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 0390c475f..c253b9c9f 100644 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -16,11 +16,13 @@ import sys from pathlib import Path +from release_versions import VERSION_PATTERN + CHANGIE = ['go', 'run', 'github.com/miniscruff/changie@v1.24.0'] def prepare(version, root, run=subprocess.run): - if not re.fullmatch(r'\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?', version): + if not re.fullmatch(VERSION_PATTERN, version): raise ValueError('Version must be X.Y.Z or X.Y.Z-rc.1 without a leading v') def git(*args): diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 4bccd95d0..203a49e8f 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -17,6 +17,8 @@ import subprocess import sys +from release_versions import VERSION_PATTERN + def require_green_ci(repo, sha): result = subprocess.run( @@ -57,7 +59,7 @@ def git(*args): if os.environ.get('GITHUB_REF_TYPE') != 'tag': raise RuntimeError('A release requires a tag push') tag = os.environ['GITHUB_REF_NAME'] - if not re.fullmatch(r'v\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?', tag): + if not re.fullmatch('v' + VERSION_PATTERN, tag): raise RuntimeError('Invalid release tag') head = git('rev-parse', 'HEAD') if git('rev-parse', f'refs/tags/{tag}^{{commit}}') != head: diff --git a/scripts/release_versions.py b/scripts/release_versions.py new file mode 100644 index 000000000..910bdcd8f --- /dev/null +++ b/scripts/release_versions.py @@ -0,0 +1,14 @@ +###################################################################### +# +# File: scripts/release_versions.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""The release interface supports stable and Python-compatible a/b/rc versions.""" + +VERSION_PATTERN = ( + r'(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)' r'(?:-(?:a|b|rc)\.(?:0|[1-9]\d*))?' +) diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py index 1b6ac0e3e..cba3572b5 100644 --- a/scripts/verify_release_tag.py +++ b/scripts/verify_release_tag.py @@ -21,6 +21,8 @@ import urllib.request from pathlib import Path +from release_versions import VERSION_PATTERN + def enforcing(value): normalized = value.strip().lower() @@ -45,7 +47,7 @@ def fetch_keys(url): def verify(tag, root, *, enforce, maintainers, fetch=fetch_keys): verified = False try: - if not re.fullmatch(r'v\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?', tag): + if not re.fullmatch('v' + VERSION_PATTERN, tag): raise ValueError('Invalid release tag') if not maintainers or any(not re.fullmatch(r'[A-Za-z0-9-]+', name) for name in maintainers): raise ValueError('Invalid maintainer list') diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py index 80885d5b2..ff646a8d7 100644 --- a/test/unit/test_release_tooling.py +++ b/test/unit/test_release_tooling.py @@ -13,7 +13,9 @@ import runpy import shutil import subprocess +import sys from pathlib import Path +from unittest.mock import patch import pytest @@ -21,7 +23,8 @@ def tooling(name): - return runpy.run_path(str(ROOT / 'scripts' / f'{name}.py')) + with patch.object(sys, 'path', [str(ROOT / 'scripts'), *sys.path]): + return runpy.run_path(str(ROOT / 'scripts' / f'{name}.py')) @pytest.fixture @@ -83,7 +86,8 @@ def run(args, **kwargs): assert git('tag', '--list').stdout == '' -def test_release_preparation_preserves_the_prerelease_version(release_repo): +@pytest.mark.parametrize('version', ['2.14.0-a.1', '2.14.0-b.1', '2.14.0-rc.1']) +def test_release_preparation_preserves_the_prerelease_version(release_repo, version): root, _ = release_repo calls = [] @@ -93,8 +97,21 @@ def run(args, **kwargs): return subprocess.CompletedProcess(args, 0, '') return subprocess.run(args, **kwargs) - tooling('prepare_release')['prepare']('2.14.0-rc.1', root, run=run) - assert calls[0][3:] == ['batch', 'v2.14.0-rc.1'] + tooling('prepare_release')['prepare'](version, root, run=run) + assert calls[0][3:] == ['batch', f'v{version}'] + + +@pytest.mark.parametrize('version', ['2.14.0-preview.1', '2.14.0-rc-1', '2.14.0-rc..1']) +def test_release_preparation_rejects_unsupported_prerelease_schemes(release_repo, version): + root, _ = release_repo + + def run(args, **kwargs): + if args[0] == 'go': + return subprocess.CompletedProcess(args, 0, '') + return subprocess.run(args, **kwargs) + + with pytest.raises(ValueError, match='X.Y.Z'): + tooling('prepare_release')['prepare'](version, root, run=run) def test_github_no_gpg_keys_response_does_not_prevent_ssh_verification(release_repo, monkeypatch): From ba86d001758aca22f15f6909dcc744abe039325f Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 15:40:11 -0600 Subject: [PATCH 05/12] fix(release): preserve historical prerelease tag syntax --- RELEASE.md | 6 ++++-- scripts/prepare_release.py | 11 ++++++++--- scripts/release_preflight.py | 2 +- scripts/release_versions.py | 14 -------------- scripts/verify_release_tag.py | 2 +- test/unit/test_release_tooling.py | 13 +++++++++++-- 6 files changed, 25 insertions(+), 23 deletions(-) delete mode 100644 scripts/release_versions.py diff --git a/RELEASE.md b/RELEASE.md index b37ba103d..6839769a2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -106,8 +106,10 @@ details out of public issues, PRs, release notes and workflow logs. consumes the unreleased fragments and updates `CHANGELOG.md`. Historical notes remain in `.changes/v2.13.4.md`; do not rewrite that archive. Review the new `## [X.Y.Z]` section and regenerate with `changie merge` after editing its batch. - Supported prerelease suffixes are `-a.N`, `-b.N` and `-rc.N`, where `N` is a - nonnegative integer without leading zeroes. This is a deliberately restricted, + Supported prerelease suffixes are `-aN`, `-bN` and `-rcN`, also accepting a dot + before `N` (`-a.N`, `-b.N`, `-rc.N`). This preserves the repository's historical + `v1.0.0-rc1` tag style. Components use ASCII digits without leading zeroes; + `N` is nonnegative. This is a deliberately restricted, Python-compatible interface, not arbitrary SemVer identifiers or build metadata. All three release entry points use the same validator. For example, run `uv run --locked --group nox nox -vs make_release_commit -- 2.14.0-rc.1`. diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index c253b9c9f..011ba7091 100644 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -16,14 +16,19 @@ import sys from pathlib import Path -from release_versions import VERSION_PATTERN - CHANGIE = ['go', 'run', 'github.com/miniscruff/changie@v1.24.0'] +VERSION_PATTERN = ( + r'(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)' + r'(?:-(?:a|b|rc)\.?(?:0|[1-9][0-9]*))?' +) def prepare(version, root, run=subprocess.run): if not re.fullmatch(VERSION_PATTERN, version): - raise ValueError('Version must be X.Y.Z or X.Y.Z-rc.1 without a leading v') + raise ValueError( + 'Version must be X.Y.Z, optionally followed by -a.N, -b.N or -rc.N ' + '(the dot before N is optional); omit the leading v' + ) def git(*args): return run( diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 203a49e8f..527e06558 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -17,7 +17,7 @@ import subprocess import sys -from release_versions import VERSION_PATTERN +from prepare_release import VERSION_PATTERN def require_green_ci(repo, sha): diff --git a/scripts/release_versions.py b/scripts/release_versions.py deleted file mode 100644 index 910bdcd8f..000000000 --- a/scripts/release_versions.py +++ /dev/null @@ -1,14 +0,0 @@ -###################################################################### -# -# File: scripts/release_versions.py -# -# Copyright 2026 Backblaze Inc. All Rights Reserved. -# -# License https://www.backblaze.com/using_b2_code.html -# -###################################################################### -"""The release interface supports stable and Python-compatible a/b/rc versions.""" - -VERSION_PATTERN = ( - r'(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)' r'(?:-(?:a|b|rc)\.(?:0|[1-9]\d*))?' -) diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py index cba3572b5..4acd4eb6c 100644 --- a/scripts/verify_release_tag.py +++ b/scripts/verify_release_tag.py @@ -21,7 +21,7 @@ import urllib.request from pathlib import Path -from release_versions import VERSION_PATTERN +from prepare_release import VERSION_PATTERN def enforcing(value): diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py index ff646a8d7..40b723960 100644 --- a/test/unit/test_release_tooling.py +++ b/test/unit/test_release_tooling.py @@ -86,7 +86,7 @@ def run(args, **kwargs): assert git('tag', '--list').stdout == '' -@pytest.mark.parametrize('version', ['2.14.0-a.1', '2.14.0-b.1', '2.14.0-rc.1']) +@pytest.mark.parametrize('version', ['2.14.0-a.1', '2.14.0-b.1', '2.14.0-rc.1', '2.14.0-rc1']) def test_release_preparation_preserves_the_prerelease_version(release_repo, version): root, _ = release_repo calls = [] @@ -101,7 +101,9 @@ def run(args, **kwargs): assert calls[0][3:] == ['batch', f'v{version}'] -@pytest.mark.parametrize('version', ['2.14.0-preview.1', '2.14.0-rc-1', '2.14.0-rc..1']) +@pytest.mark.parametrize( + 'version', ['2.14.0-preview.1', '2.14.0-rc-1', '2.14.0-rc..1', '2.1\u0664.0'] +) def test_release_preparation_rejects_unsupported_prerelease_schemes(release_repo, version): root, _ = release_repo @@ -114,6 +116,13 @@ def run(args, **kwargs): tooling('prepare_release')['prepare'](version, root, run=run) +def test_version_error_explains_all_supported_stages(release_repo): + root, _ = release_repo + with pytest.raises(ValueError) as error: + tooling('prepare_release')['prepare']('unsupported', root) + assert all(stage in str(error.value) for stage in ('a.N', 'b.N', 'rc.N')) + + def test_github_no_gpg_keys_response_does_not_prevent_ssh_verification(release_repo, monkeypatch): root, git = release_repo key = root / 'fixture-key' From 6fd25ee9ca39b849f10268072ee4a5f7cd7ba5c6 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 15:50:45 -0600 Subject: [PATCH 06/12] fix(release): protect security reads and preserve prerelease publishing --- .github/workflows/cd.yml | 5 ++-- RELEASE.md | 30 ++++++++++++++-------- pyproject.toml | 1 + scripts/release_preflight.py | 33 +++++++++++++++++++------ scripts/release_security.py | 7 +++++- test/unit/test_release_tooling.py | 41 ++++++++++++++++++++++++++++++- uv.lock | 2 ++ 7 files changed, 98 insertions(+), 21 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 2bf836eb2..086689648 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -26,6 +26,7 @@ jobs: release-gates: runs-on: ubuntu-latest timeout-minutes: 10 + environment: release-security permissions: contents: read actions: read @@ -114,7 +115,7 @@ jobs: publish-pypi: needs: build - if: github.event_name == 'push' && needs.build.outputs.prerelease == 'false' + if: github.event_name == 'push' runs-on: ubuntu-latest timeout-minutes: 10 environment: @@ -137,7 +138,7 @@ jobs: needs: [build, publish-pypi] if: >- always() && github.event_name == 'push' && needs.build.result == 'success' && - (needs.publish-pypi.result == 'success' || needs.build.outputs.prerelease == 'true') + needs.publish-pypi.result == 'success' runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/RELEASE.md b/RELEASE.md index 6839769a2..b0f643417 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -19,10 +19,14 @@ Towncrier in the `release` dependency group. This does not add a runtime depende the old publishing credential; remove that credential after the migration is verified. - The publish job has `id-token: write` and explicitly enables PEP 740 attestations. It downloads built distributions and does not build code or install build dependencies. -- Ensure the release gate can read Code scanning alerts, Dependabot alerts and - Secret scanning alerts. If `GITHUB_TOKEN` cannot read all three, configure - `RELEASE_SECURITY_TOKEN` with those read permissions (prefer an installation - token managed by the organization). An unreadable or malformed source **holds** +- Create a `release-security` GitHub environment with deployment rules allowing + only `master` and `v*` tags, required security-maintainer approval, and self-review + disabled. Configure these protections **before adding any credential**. + Ensure its gate can read Code scanning alerts, Dependabot alerts and Secret + scanning alerts. If `GITHUB_TOKEN` cannot read all three, store + `RELEASE_SECURITY_TOKEN` only as a secret of this protected environment (prefer + an installation token managed by the organization). Never store a copy as a + repository/organization secret; remove such a copy if it exists. An unreadable or malformed source **holds** the release; disabled scanners are not silently treated as clean. - Select and review the intended maintainer accounts in `.github/signing-maintainers`: `goanpeca`, `ffumero2003`, `sophiecarreras`. They are GitHub usernames, not fixture keys. @@ -60,7 +64,9 @@ uv run --group nox python scripts/release_security.py Backblaze/b2-sdk-python It holds for medium/high/critical CodeQL security alerts, medium/high/critical Dependabot alerts with an available fix, and every open secret-scanning alert -(that API has no severity field). Low alerts and dependencies without a patched +(that API has no severity field). Code-scanning alerts from other tools are outside +this automated CodeQL policy. Secret-scanning requests set `hide_secret=true`. +Low alerts and dependencies without a patched version do not hold this gate. The output contains source counts only. Resolve or have an authorized owner dismiss findings in their source system with a reason; the gate has no finding waiver flag. An API error is `UNVERIFIED` and exits nonzero. @@ -153,7 +159,7 @@ details out of public issues, PRs, release notes and workflow logs. After the dispatch-enabled `cd.yml` is on the default branch: ```sh -gh workflow run cd.yml --repo Backblaze/b2-sdk-python --ref release/vX.Y.Z -f dry_run=true +gh workflow run cd.yml --repo Backblaze/b2-sdk-python --ref master -f dry_run=true ``` The dry run checks current master CI and the live security sources, assembles a @@ -164,7 +170,9 @@ Neither the PyPI publisher nor `gh release create/edit/upload` runs on dispatch; configuration or attestation upload; those are only exercised by an actual approved publish. TestPyPI is not configured by this workflow and dispatch does not publish there. GitHub may reject dispatch before the workflow trigger exists on master; record -that as a verification limit rather than bypassing the gate. +that as a verification limit rather than bypassing the gate. The protected +security environment intentionally rejects dispatches on release/feature branches; +its privileged token is not available to unmerged workflow code. ## Tag and publish (maintainer only) @@ -187,9 +195,11 @@ Configure `gpg.format=ssh` and `user.signingkey` for SSH, or the GPG signing key GPG, before `git tag -s`. The tag workflow requires successful master and tagged commit CI, master ancestry, the configured signature gate, a clean security gate, matching Changie version/notes and no remaining fragments before publishing. -Stable tags publish to PyPI through OIDC with PEP 740 attestations. Prerelease tags -create a GitHub prerelease without a PyPI upload, matching the prior stable-only -publication policy. The publishing job requires the protected `pypi` environment. +Stable and supported prerelease tags publish to PyPI through OIDC with PEP 740 +attestations, preserving uploads for the established digit-terminated prerelease +tag style. Prereleases are now correctly marked as GitHub prereleases instead of +using the previous last-character classification. The publishing job requires the +protected `pypi` environment for both stable and prerelease uploads. After successful PyPI upload, the workflow creates the GitHub release with the exact changelog section. Re-running the GitHub-release job edits existing notes diff --git a/pyproject.toml b/pyproject.toml index 99337d228..cd706ca68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ lint = [ "setuptools<81", # liccheck 0.9.2 imports pkg_resources, removed in setuptools 81 ] test = [ + {include-group = "release"}, "pip>=26.2.2", "coverage>=7.13.5,<8", "pytest>=9.0.3,<10", diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 527e06558..de0115c74 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -20,6 +20,25 @@ from prepare_release import VERSION_PATTERN +class ReleaseHold(RuntimeError): + """An actionable reason constructed by this gate, never remote error content.""" + + +def failure_reason(error): + if isinstance(error, ReleaseHold): + reason = str(error) + elif isinstance(error, subprocess.CalledProcessError): + if error.cmd[:2] == ['git', 'merge-base']: + reason = 'Release commit is not on master history' + elif error.cmd[0] == 'gh': + reason = 'Cannot read required CI metadata' + else: + reason = 'Cannot fetch or resolve the release and master revisions' + else: + reason = 'Required release or CI metadata is missing or malformed' + return f'Release preflight failed: {reason}' + + def require_green_ci(repo, sha): result = subprocess.run( ['gh', 'api', f'repos/{repo}/actions/workflows/ci.yml/runs?head_sha={sha}&per_page=100'], @@ -33,7 +52,7 @@ def require_green_ci(repo, sha): if run['head_sha'] == sha and run['head_branch'] == 'master' and run['event'] == 'push' ] if not runs or runs[0]['status'] != 'completed' or runs[0]['conclusion'] != 'success': - raise RuntimeError( + raise ReleaseHold( 'The latest master CI run for this commit must have completed successfully' ) @@ -41,7 +60,7 @@ def require_green_ci(repo, sha): def main(): repo = os.environ['GITHUB_REPOSITORY'] if repo != 'Backblaze/b2-sdk-python': - raise RuntimeError('Release workflow is restricted to Backblaze/b2-sdk-python') + raise ReleaseHold('Release workflow is restricted to Backblaze/b2-sdk-python') subprocess.run(['git', 'fetch', '--no-tags', 'origin', 'master'], check=True) def git(*args): @@ -53,17 +72,17 @@ def git(*args): require_green_ci(repo, master) if os.environ['GITHUB_EVENT_NAME'] == 'workflow_dispatch': if os.environ.get('INPUT_DRY_RUN') != 'true': - raise RuntimeError('Manual dispatch supports dry_run=true only; it never publishes') + raise ReleaseHold('Manual dispatch supports dry_run=true only; it never publishes') print('Non-publishing dry run; master CI is green.') return if os.environ.get('GITHUB_REF_TYPE') != 'tag': - raise RuntimeError('A release requires a tag push') + raise ReleaseHold('A release requires a tag push') tag = os.environ['GITHUB_REF_NAME'] if not re.fullmatch('v' + VERSION_PATTERN, tag): - raise RuntimeError('Invalid release tag') + raise ReleaseHold('Invalid release tag') head = git('rev-parse', 'HEAD') if git('rev-parse', f'refs/tags/{tag}^{{commit}}') != head: - raise RuntimeError('Checked-out revision differs from the release tag') + raise ReleaseHold('Checked-out revision differs from the release tag') subprocess.run(['git', 'merge-base', '--is-ancestor', head, master], check=True) require_green_ci(repo, head) print('Release tag is on master history and both release and master CI are green.') @@ -74,4 +93,4 @@ def git(*args): main() except (KeyError, ValueError, RuntimeError, subprocess.SubprocessError) as error: # Do not echo API payloads or credential-bearing stderr. - sys.exit(f'Release preflight failed ({type(error).__name__})') + sys.exit(failure_reason(error)) diff --git a/scripts/release_security.py b/scripts/release_security.py index 21b0c9e06..cc9fd3d42 100644 --- a/scripts/release_security.py +++ b/scripts/release_security.py @@ -38,7 +38,10 @@ def check(repo, api=github_api): holds = [] for source in ('dependabot', 'code-scanning', 'secret-scanning'): try: - items = api(f'repos/{repo}/{source}/alerts?state=open&per_page=100') + endpoint = f'repos/{repo}/{source}/alerts?state=open&per_page=100' + if source == 'secret-scanning': + endpoint += '&hide_secret=true' + items = api(endpoint) if not isinstance(items, list) or not all(isinstance(item, dict) for item in items): raise ValueError('Invalid alert response') count = 0 @@ -51,6 +54,8 @@ def check(repo, api=github_api): item['security_vulnerability']['first_patched_version'] ) elif source == 'code-scanning': + if item['tool']['name'] != 'CodeQL': + continue rule = item['rule'] severity = rule.get('security_severity_level') if severity not in BLOCKING | {'low', None}: diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py index 40b723960..d62ee9f05 100644 --- a/test/unit/test_release_tooling.py +++ b/test/unit/test_release_tooling.py @@ -18,6 +18,7 @@ from unittest.mock import patch import pytest +import yaml ROOT = Path(__file__).parents[2] @@ -204,10 +205,48 @@ def test_security_gate_does_not_block_low_or_unpatched_dependabot_alerts(): @pytest.mark.parametrize('severity', ['medium', 'high', 'critical']) def test_security_gate_blocks_code_scanning_alerts(severity): - source = {'rule': {'security_severity_level': severity}} + source = {'tool': {'name': 'CodeQL'}, 'rule': {'security_severity_level': severity}} assert tooling('release_security')['check']('owner/repo', api=alerts(code=[source])) +def test_security_gate_applies_the_codeql_policy_to_codeql_only(): + source = {'tool': {'name': 'Other scanner'}, 'rule': {'security_severity_level': 'high'}} + assert tooling('release_security')['check']('owner/repo', api=alerts(code=[source])) == [] + + +def test_security_gate_requests_secret_metadata_without_secret_values(): + endpoints = [] + + def api(endpoint): + endpoints.append(endpoint) + return [] + + assert tooling('release_security')['check']('owner/repo', api=api) == [] + assert any('/secret-scanning/' in url and 'hide_secret=true' in url for url in endpoints) + + +def test_release_security_credentials_have_a_protected_environment_boundary(): + workflow = yaml.load((ROOT / '.github/workflows/cd.yml').read_text(), Loader=yaml.BaseLoader) + gates = workflow['jobs']['release-gates'] + assert gates.get('environment') == 'release-security' + assert gates['permissions']['contents'] == 'read' + assert 'id-token' not in gates['permissions'] + for name in ('publish-pypi', 'github-release'): + assert "github.event_name == 'push'" in workflow['jobs'][name]['if'] + + +def test_supported_prerelease_tags_retain_the_pypi_publish_path(): + workflow = yaml.load((ROOT / '.github/workflows/cd.yml').read_text(), Loader=yaml.BaseLoader) + condition = workflow['jobs']['publish-pypi']['if'] + assert condition == "github.event_name == 'push'" + + +def test_release_preflight_retains_safe_actionable_failure_reasons(): + module = tooling('release_preflight') + hold = module['ReleaseHold']('Current master CI is not successful') + assert 'Current master CI is not successful' in module['failure_reason'](hold) + + def test_security_gate_blocks_open_secret_scanning_alerts(): assert tooling('release_security')['check']('owner/repo', api=alerts(secrets=[{}])) diff --git a/uv.lock b/uv.lock index 5bf93507b..37f1c235c 100644 --- a/uv.lock +++ b/uv.lock @@ -118,6 +118,7 @@ test = [ { name = "pytest-watcher" }, { name = "pytest-xdist" }, { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, + { name = "pyyaml" }, { name = "responses" }, { name = "tenacity" }, { name = "tqdm" }, @@ -173,6 +174,7 @@ test = [ { name = "pytest-watcher", specifier = ">=0.6.3,<1" }, { name = "pytest-xdist", specifier = ">=3.8.0,<4" }, { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'", specifier = ">=311,<312" }, + { name = "pyyaml", specifier = ">=6.0.3,<7" }, { name = "responses", specifier = ">=0.26.0,<1" }, { name = "tenacity", specifier = ">=9.1.4,<10" }, { name = "tqdm", specifier = ">=4.5.0,<5.0.0" }, From ca676ce6f4381def4bc5a41e96fd23fd84199f88 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 15:54:40 -0600 Subject: [PATCH 07/12] docs(release): require protected tag creation and independent approval --- RELEASE.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index b0f643417..bc38fd78c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,6 +9,13 @@ Towncrier in the `release` dependency group. This does not add a runtime depende ## One-time setup +- A repository administrator must configure an **active tag ruleset** for `v*` + before enabling either protected release environment. Restrict tag creation and + updates to the release-maintainer team through an explicit bypass list; do not + grant a blanket write-role bypass. Protect published tags against update/deletion + and keep `master` behind required review and CI. Review the ruleset whenever the + release-maintainer membership changes. These are repository settings, not rules + a tag-controlled workflow can securely enforce about itself. - Install Git, GitHub CLI, Go, uv and Python 3.14. Signed GPG tags also need GnuPG; SSH-signed tags need Git 2.34+ and OpenSSH. Authenticate `gh` with this repository. - Configure a PyPI **Trusted Publisher** for project `b2sdk`, owner `Backblaze`, @@ -31,6 +38,28 @@ Towncrier in the `release` dependency group. This does not add a runtime depende - Select and review the intended maintainer accounts in `.github/signing-maintainers`: `goanpeca`, `ffumero2003`, `sophiecarreras`. They are GitHub usernames, not fixture keys. +### Environment approval is an independent trust check + +Before approving **either** `release-security` or `pypi`, inspect the exact tag +commit and its release PR from a trusted checkout. Confirm its master ancestry and +successful CI independently of the candidate workflow: + +```sh +TAG=vX.Y.Z +git fetch origin master +git fetch origin tag "$TAG" +sha=$(git rev-parse "$TAG^{commit}") +git merge-base --is-ancestor "$sha" origin/master +git log -1 --oneline "$sha" +gh run list --repo Backblaze/b2-sdk-python --workflow ci.yml --branch master --commit "$sha" --limit 1 +``` + +Do not approve solely because the deployment displays a `v*` tag name. A tag can +point to unmerged code, including a modified workflow. The external tag ruleset +and the approver's independent ancestry/CI check protect the credential and OIDC +boundaries; the in-workflow preflight provides an additional normal-path check. +For a master dry run, confirm the selected master commit and CI instead. + ### Enroll signing keys before enforcement There is no enrolled-key baseline yet. Each maintainer must create a personal GPG @@ -172,7 +201,9 @@ publish. TestPyPI is not configured by this workflow and dispatch does not publi GitHub may reject dispatch before the workflow trigger exists on master; record that as a verification limit rather than bypassing the gate. The protected security environment intentionally rejects dispatches on release/feature branches; -its privileged token is not available to unmerged workflow code. +tag deployments additionally require the tag ruleset and independent environment +approval described above. Environment ref patterns alone do not establish that a +tag's commit was merged. ## Tag and publish (maintainer only) From 2ce060b98ff08f095efb49662df6d3f3ed5c255c Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 16:02:05 -0600 Subject: [PATCH 08/12] chore(release): import the merged 2.13.5 notes --- .changes/README.md | 2 +- .../unreleased/sdkharness-legal-hold-delete-refusal.yaml | 5 ----- .changes/{v2.13.4.md => v2.13.5.md} | 8 ++++++++ .github/workflows/ci.yml | 2 +- RELEASE.md | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) delete mode 100644 .changes/unreleased/sdkharness-legal-hold-delete-refusal.yaml rename .changes/{v2.13.4.md => v2.13.5.md} (99%) diff --git a/.changes/README.md b/.changes/README.md index 7e1200841..191a46de3 100644 --- a/.changes/README.md +++ b/.changes/README.md @@ -3,7 +3,7 @@ Add a fragment with `go run github.com/miniscruff/changie@v1.24.0 new`. Unreleased YAML fragments belong in `unreleased/`. -`v2.13.4.md` preserves the complete changelog history through v2.13.4, imported +`v2.13.5.md` preserves the complete changelog history through v2.13.5, imported without changing any released section. Future batches create one file per version. Keep this historical archive intact. Regenerate `CHANGELOG.md` with `go run github.com/miniscruff/changie@v1.24.0 merge` after editing a batch or header. diff --git a/.changes/unreleased/sdkharness-legal-hold-delete-refusal.yaml b/.changes/unreleased/sdkharness-legal-hold-delete-refusal.yaml deleted file mode 100644 index 24f75fc73..000000000 --- a/.changes/unreleased/sdkharness-legal-hold-delete-refusal.yaml +++ /dev/null @@ -1,5 +0,0 @@ -{ - "kind": "Infrastructure", - "body": "Make the sdkharness legal-hold delete check accept B2's `401 access_denied` response while retaining older simulator refusal shapes.", - "time": "2026-11-13T19:52:04Z" -} diff --git a/.changes/v2.13.4.md b/.changes/v2.13.5.md similarity index 99% rename from .changes/v2.13.4.md rename to .changes/v2.13.5.md index ad6d74bef..73d95ac29 100644 --- a/.changes/v2.13.4.md +++ b/.changes/v2.13.5.md @@ -1,3 +1,11 @@ +## [2.13.5](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.5) - 2026-11-17 + + +### Infrastructure + +- Make the sdkharness legal-hold delete check accept B2's `401 access_denied` response while retaining older simulator refusal shapes. + + ## [2.13.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.4) - 2026-11-12 ### Fixed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07b4e4b35..c8a1d2d9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: # caf is intentional escaped Unicode test data. run: | uvx codespell==2.4.1 --ignore-words-list=datas,re-use,caf --skip='.git,.nox,.venv,build,dist,uv.lock,CHANGELOG.md' - uvx codespell==2.4.1 --check-hidden .changes --skip='.changes/v2.13.4.md' + uvx codespell==2.4.1 --check-hidden .changes --skip='.changes/v2.13.5.md' changelog-check: timeout-minutes: 10 runs-on: ubuntu-latest diff --git a/RELEASE.md b/RELEASE.md index bc38fd78c..acd4895d1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -139,7 +139,7 @@ details out of public issues, PRs, release notes and workflow logs. rejects an existing tag; runs `changie batch vX.Y.Z` then `changie merge`; and stops. It never commits, tags or publishes. It creates `.changes/vX.Y.Z.md`, consumes the unreleased fragments and updates `CHANGELOG.md`. Historical notes - remain in `.changes/v2.13.4.md`; do not rewrite that archive. Review the new + remain in `.changes/v2.13.5.md`; do not rewrite that archive. Review the new `## [X.Y.Z]` section and regenerate with `changie merge` after editing its batch. Supported prerelease suffixes are `-aN`, `-bN` and `-rcN`, also accepting a dot before `N` (`-a.N`, `-b.N`, `-rc.N`). This preserves the repository's historical From 0b6cda7f09067a7069283992d94e596a3b0aa65a Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 16:08:24 -0600 Subject: [PATCH 09/12] docs(release): point contributors to the current archive --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c18263f3..86dbbaa47 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ A `-changelog` label can waive adding a fragment; validation still runs. Lockfil updates and release PRs that batch fragments have their own checks. Release managers use `changie batch` and `changie merge` through the Nox release -session. See [RELEASE.md](RELEASE.md) for the complete process. The `.changes/v2.13.4.md` +session. See [RELEASE.md](RELEASE.md) for the complete process. The `.changes/v2.13.5.md` archive preserves all earlier release notes; do not rewrite that history. ## Developer Info From 7d73ec9872fe3d2d10401280b5faef049bee7f73 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 17:34:17 -0600 Subject: [PATCH 10/12] fix(release): handle released trees and preserve tag verification --- .github/workflows/cd.yml | 5 ++ RELEASE.md | 12 ++-- scripts/check_changelog.py | 5 +- scripts/prepare_release.py | 5 +- scripts/verify_release_tag.py | 25 +++++++- test/unit/test_release_tooling.py | 102 ++++++++++++++++++++++++++++-- 6 files changed, 134 insertions(+), 20 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 086689648..1669d59a0 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -41,6 +41,11 @@ jobs: GH_TOKEN: ${{ github.token }} INPUT_DRY_RUN: ${{ inputs.dry_run }} run: python scripts/release_preflight.py + - name: Restore annotated release tag + if: github.event_name == 'push' + env: + TAG: ${{ github.ref_name }} + run: git fetch --force --no-tags origin "refs/tags/$TAG:refs/tags/$TAG" - name: Verify maintainer tag signature if: github.event_name == 'push' env: diff --git a/RELEASE.md b/RELEASE.md index acd4895d1..89bb0cb74 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -65,10 +65,12 @@ For a master dry run, confirm the selected master commit and CI instead. There is no enrolled-key baseline yet. Each maintainer must create a personal GPG or SSH signing key, register its public key with their GitHub account and configure Git to sign tags. Review fingerprints with the maintainer through an independent -channel. The workflow resolves public keys from `https://github.com/.gpg` -and `.keys` at check time into a temporary keyring/allowed-signers file, and runs +channel. The workflow resolves GPG keys from `https://github.com/.gpg` +and SSH signing keys from `https://api.github.com/users//ssh_signing_keys` +at check time into a temporary keyring/allowed-signers file, and runs `git verify-tag`. Do not commit a private key or test key to the maintainer list. -An SSH signing key must be present in the public `.keys` response used by the verifier. +Register SSH keys as **Signing Keys** on GitHub. The `.keys` endpoint lists +authentication keys and is not used to authorize release signatures. `ENFORCE_SIGNED_TAGS` is a repository variable. Unset or `false` means **warn-only**: an unsigned, unknown-key or unverifiable tag is reported but does not hold the @@ -143,8 +145,8 @@ details out of public issues, PRs, release notes and workflow logs. `## [X.Y.Z]` section and regenerate with `changie merge` after editing its batch. Supported prerelease suffixes are `-aN`, `-bN` and `-rcN`, also accepting a dot before `N` (`-a.N`, `-b.N`, `-rc.N`). This preserves the repository's historical - `v1.0.0-rc1` tag style. Components use ASCII digits without leading zeroes; - `N` is nonnegative. This is a deliberately restricted, + `v1.0.0-rc1` tag style. Numeric release components accept ASCII digits, including + leading zeroes; `N` is nonnegative. This is a deliberately restricted, Python-compatible interface, not arbitrary SemVer identifiers or build metadata. All three release entry points use the same validator. For example, run `uv run --locked --group nox nox -vs make_release_commit -- 2.14.0-rc.1`. diff --git a/scripts/check_changelog.py b/scripts/check_changelog.py index 168dd1a8a..86feca81f 100644 --- a/scripts/check_changelog.py +++ b/scripts/check_changelog.py @@ -40,6 +40,7 @@ def main(): parser.add_argument('--allow-missing-fragment', action='store_true') args = parser.parse_args() root = Path.cwd() + has_fragments = False for path in (root / '.changes/unreleased').iterdir(): if path.name == '.gitkeep': continue @@ -48,12 +49,14 @@ def main(): if path.stat().st_size > 16 * 1024: raise ValueError('Fragment exceeds 16 KiB') validate_fragment(yaml.safe_load(path.read_text())) + has_fragments = True merged = subprocess.run( [*CHANGIE, 'merge', '--dry-run'], check=True, capture_output=True, text=True ) if merged.stdout != (root / 'CHANGELOG.md').read_text(): raise ValueError('CHANGELOG.md differs from changie merge; regenerate it') - subprocess.run([*CHANGIE, 'batch', 'patch', '--dry-run'], check=True, capture_output=True) + if has_fragments: + subprocess.run([*CHANGIE, 'batch', 'patch', '--dry-run'], check=True, capture_output=True) if args.base and not args.allow_missing_fragment: diff = subprocess.run( ['git', 'diff', '--name-status', f'{args.base}...HEAD'], diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 011ba7091..29a9943a5 100644 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -17,10 +17,7 @@ from pathlib import Path CHANGIE = ['go', 'run', 'github.com/miniscruff/changie@v1.24.0'] -VERSION_PATTERN = ( - r'(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)' - r'(?:-(?:a|b|rc)\.?(?:0|[1-9][0-9]*))?' -) +VERSION_PATTERN = r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:a|b|rc)\.?(?:0|[1-9][0-9]*))?' def prepare(version, root, run=subprocess.run): diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py index 4acd4eb6c..0ecbe6185 100644 --- a/scripts/verify_release_tag.py +++ b/scripts/verify_release_tag.py @@ -11,6 +11,7 @@ from __future__ import annotations +import json import os import re import shutil @@ -67,9 +68,27 @@ def verify(tag, root, *, enforce, maintainers, fetch=fetch_keys): capture_output=True, check=False, ) - for key in fetch(f'https://github.com/{name}.keys').splitlines(): - if key.strip(): - ssh_keys.append(f'{name} {key}') + page = 1 + while True: + keys = json.loads( + fetch( + f'https://api.github.com/users/{name}/ssh_signing_keys?per_page=100&page={page}' + ) + ) + if not isinstance(keys, list): + raise ValueError('Invalid SSH signing-key response') + for item in keys: + key = item.get('key') if isinstance(item, dict) else None + if ( + not isinstance(key, str) + or not key.strip() + or len(key.splitlines()) != 1 + ): + raise ValueError('Invalid SSH signing key') + ssh_keys.append(f'{name} {key.strip()}') + if len(keys) < 100: + break + page += 1 signers = home / 'allowed_signers' signers.write_text('\n'.join(ssh_keys) + '\n') commands = [ diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py index d62ee9f05..5cab6a9c9 100644 --- a/test/unit/test_release_tooling.py +++ b/test/unit/test_release_tooling.py @@ -11,6 +11,7 @@ import json import runpy +import shlex import shutil import subprocess import sys @@ -102,6 +103,20 @@ def run(args, **kwargs): assert calls[0][3:] == ['batch', f'v{version}'] +def test_release_preparation_preserves_zero_padded_components(release_repo): + root, _ = release_repo + calls = [] + + def run(args, **kwargs): + if args[0] == 'go': + calls.append(args) + return subprocess.CompletedProcess(args, 0, '') + return subprocess.run(args, **kwargs) + + tooling('prepare_release')['prepare']('02.13.05', root, run=run) + assert [call[3:] for call in calls] == [['batch', 'v02.13.05'], ['merge']] + + @pytest.mark.parametrize( 'version', ['2.14.0-preview.1', '2.14.0-rc-1', '2.14.0-rc..1', '2.1\u0664.0'] ) @@ -150,8 +165,8 @@ def run(args, **kwargs): return real_run(args, **kwargs) def fetch(url): - if url.endswith('.keys'): - return public_key + if '/ssh_signing_keys?' in url: + return json.dumps([{'key': public_key.strip()}]) return ( "-----BEGIN PGP PUBLIC KEY BLOCK-----\nNote: This user hasn't uploaded any GPG keys.\n" ) @@ -241,6 +256,33 @@ def test_supported_prerelease_tags_retain_the_pypi_publish_path(): assert condition == "github.event_name == 'push'" +def test_release_workflow_restores_the_origin_tag_object(release_repo): + root, git = release_repo + origin = root / 'origin.git' + git('init', '--bare', str(origin)) + git('remote', 'add', 'origin', str(origin)) + tag = 'v2.13.5' + git('tag', '-a', tag, '-m', 'Annotated fixture') + tag_object = git('rev-parse', tag).stdout.strip() + git('push', 'origin', f'refs/tags/{tag}') + # Reproduce checkout's replacement of the tag object with the peeled commit. + git('update-ref', f'refs/tags/{tag}', git('rev-parse', 'HEAD').stdout.strip()) + assert git('cat-file', '-t', tag).stdout.strip() == 'commit' + + workflow = yaml.load((ROOT / '.github/workflows/cd.yml').read_text(), Loader=yaml.BaseLoader) + steps = workflow['jobs']['release-gates']['steps'] + for step in steps: + if step.get('name') == 'Verify maintainer tag signature': + break + if step.get('name') == 'Restore annotated release tag': + assert step['if'] == "github.event_name == 'push'" + assert step['env']['TAG'] == '${{ github.ref_name }}' + command = [arg.replace('$TAG', tag) for arg in shlex.split(step['run'])] + subprocess.run(command, cwd=root, check=True, capture_output=True) + assert git('rev-parse', tag).stdout.strip() == tag_object + assert git('cat-file', '-t', tag).stdout.strip() == 'tag' + + def test_release_preflight_retains_safe_actionable_failure_reasons(): module = tooling('release_preflight') hold = module['ReleaseHold']('Current master CI is not successful') @@ -282,7 +324,10 @@ def test_unsigned_tag_is_warning_only_until_enrolled(release_repo): assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=lambda url: '') -def test_ssh_signed_tag_requires_an_enrolled_key(release_repo, tmp_path): +@pytest.mark.parametrize('signing_key_enrolled', [False, True]) +def test_ssh_signed_tag_requires_an_enrolled_signing_key( + release_repo, tmp_path, signing_key_enrolled +): root, git = release_repo # Generated fixture material stays in pytest's temporary directory. key = tmp_path / 'fixture-signing-key' @@ -303,16 +348,27 @@ def test_ssh_signed_tag_requires_an_enrolled_key(release_repo, tmp_path): # Key files are untracked fixtures; signature verification itself does not mutate the repo. def fetch(url): - return public_key if url.endswith('.keys') else '' + if url.endswith('.keys'): + return '' if signing_key_enrolled else public_key + if '/ssh_signing_keys?' in url: + return json.dumps([{'key': public_key.strip()}] if signing_key_enrolled else []) + return '' + + assert ( + verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch) + is signing_key_enrolled + ) - assert verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch) - assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=lambda url: '') + def no_keys(url): + return '[]' if '/ssh_signing_keys?' in url else '' + + assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=no_keys) # A pre-existing local/global signer configuration cannot expand enrollment. external = root / 'external-signers' external.write_text(f'external {public_key}') git('config', 'gpg.ssh.allowedSignersFile', str(external)) - assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=lambda url: '') + assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=no_keys) @pytest.mark.parametrize( @@ -329,6 +385,38 @@ def test_changie_fragment_validation_accepts_a_complete_entry(): ) +@pytest.mark.parametrize('has_fragment', [False, True]) +@pytest.mark.parametrize('base_diff', [None, 'A\t.changes/v2.13.6.md\nM\tCHANGELOG.md\n']) +def test_changelog_check_batches_only_when_fragments_exist( + tmp_path, monkeypatch, has_fragment, base_diff +): + unreleased = tmp_path / '.changes/unreleased' + unreleased.mkdir(parents=True) + (unreleased / '.gitkeep').touch() + if has_fragment: + (unreleased / 'entry.yaml').write_text( + 'kind: Fixed\nbody: Correct a release check.\ntime: 2026-11-17T00:00:00Z\n' + ) + changelog = '## [2.13.6]\n' + (tmp_path / 'CHANGELOG.md').write_text(changelog) + calls = [] + + def run(args, **kwargs): + calls.append(args) + if 'batch' in args and not has_fragment: + raise subprocess.CalledProcessError(1, args, stderr='no unreleased changes found') + output = base_diff if args[:2] == ['git', 'diff'] else changelog + return subprocess.CompletedProcess(args, 0, output) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + sys, 'argv', ['check_changelog', *(['--base', 'base'] if base_diff else [])] + ) + monkeypatch.setattr(subprocess, 'run', run) + tooling('check_changelog')['main']() + assert any('batch' in call for call in calls) is has_fragment + + @pytest.mark.parametrize( 'status, conclusion, branch, event', [ From 84861fc93d9a85dae76fba99c9dc719e45f8a238 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 17:39:22 -0600 Subject: [PATCH 11/12] fix(release): bind restored tags to the workflow commit --- RELEASE.md | 5 +++-- scripts/verify_release_tag.py | 9 +++++++++ test/unit/test_release_tooling.py | 8 ++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 89bb0cb74..48c04f529 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -74,8 +74,9 @@ authentication keys and is not used to authorize release signatures. `ENFORCE_SIGNED_TAGS` is a repository variable. Unset or `false` means **warn-only**: an unsigned, unknown-key or unverifiable tag is reported but does not hold the -release. Once every intended release signer is enrolled and a signed-tag check has -been verified, a repository administrator enables enforcement: +release. A restored tag pointing to a different commit than the checkout always +blocks, regardless of this variable. Once every intended release signer is enrolled +and a signed-tag check has been verified, a repository administrator enables enforcement: ```sh gh variable set ENFORCE_SIGNED_TAGS --repo Backblaze/b2-sdk-python --body true diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py index 0ecbe6185..1954b2d72 100644 --- a/scripts/verify_release_tag.py +++ b/scripts/verify_release_tag.py @@ -50,6 +50,15 @@ def verify(tag, root, *, enforce, maintainers, fetch=fetch_keys): try: if not re.fullmatch('v' + VERSION_PATTERN, tag): raise ValueError('Invalid release tag') + commits = [ + subprocess.run( + ['git', 'rev-parse', ref], cwd=root, check=True, capture_output=True, text=True + ).stdout.strip() + for ref in (f'refs/tags/{tag}^{{commit}}', 'HEAD') + ] + if commits[0] != commits[1]: + print('::error::Restored tag does not point to the checked-out release commit.') + return False if not maintainers or any(not re.fullmatch(r'[A-Za-z0-9-]+', name) for name in maintainers): raise ValueError('Invalid maintainer list') with tempfile.TemporaryDirectory(prefix='b2sdk-signers-') as temporary: diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py index 5cab6a9c9..447cdbf0b 100644 --- a/test/unit/test_release_tooling.py +++ b/test/unit/test_release_tooling.py @@ -370,6 +370,14 @@ def no_keys(url): git('config', 'gpg.ssh.allowedSignersFile', str(external)) assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=no_keys) + if signing_key_enrolled: + # A valid signature on a different commit cannot authorize this checkout. + git('commit', '--allow-empty', '-m', 'Different workflow commit') + for enforce in (False, True): + assert not verify( + 'v2.13.5', root, enforce=enforce, maintainers=['example'], fetch=fetch + ) + @pytest.mark.parametrize( 'value', [None, {}, {'kind': 'Fixed'}, {'kind': 'Unknown', 'body': 'Entry.'}] From 4f389e4a946bfb27be6adfa544ab10a34c82a1f1 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Tue, 17 Nov 2026 17:42:51 -0600 Subject: [PATCH 12/12] fix(release): verify the restored tag against the event SHA --- scripts/verify_release_tag.py | 4 +++- test/unit/test_release_tooling.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py index 1954b2d72..e82985c6a 100644 --- a/scripts/verify_release_tag.py +++ b/scripts/verify_release_tag.py @@ -56,7 +56,9 @@ def verify(tag, root, *, enforce, maintainers, fetch=fetch_keys): ).stdout.strip() for ref in (f'refs/tags/{tag}^{{commit}}', 'HEAD') ] - if commits[0] != commits[1]: + if commits[0] != commits[1] or ( + os.getenv('GITHUB_SHA') and commits[0] != os.environ['GITHUB_SHA'] + ): print('::error::Restored tag does not point to the checked-out release commit.') return False if not maintainers or any(not re.fullmatch(r'[A-Za-z0-9-]+', name) for name in maintainers): diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py index 447cdbf0b..f28fcb54b 100644 --- a/test/unit/test_release_tooling.py +++ b/test/unit/test_release_tooling.py @@ -30,7 +30,10 @@ def tooling(name): @pytest.fixture -def release_repo(tmp_path): +def release_repo(tmp_path, monkeypatch): + # Temporary repositories have their own commits, independent of the CI checkout. + monkeypatch.delenv('GITHUB_SHA', raising=False) + def git(*args): return subprocess.run( ['git', *args], cwd=tmp_path, check=True, capture_output=True, text=True @@ -326,7 +329,7 @@ def test_unsigned_tag_is_warning_only_until_enrolled(release_repo): @pytest.mark.parametrize('signing_key_enrolled', [False, True]) def test_ssh_signed_tag_requires_an_enrolled_signing_key( - release_repo, tmp_path, signing_key_enrolled + release_repo, tmp_path, monkeypatch, signing_key_enrolled ): root, git = release_repo # Generated fixture material stays in pytest's temporary directory. @@ -371,6 +374,15 @@ def no_keys(url): assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=no_keys) if signing_key_enrolled: + monkeypatch.setenv('GITHUB_SHA', git('rev-parse', 'HEAD').stdout.strip()) + assert verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch) + workflow_commit = git('commit-tree', 'HEAD^{tree}', '-m', 'Workflow fixture').stdout.strip() + monkeypatch.setenv('GITHUB_SHA', workflow_commit) + for enforce in (False, True): + assert not verify( + 'v2.13.5', root, enforce=enforce, maintainers=['example'], fetch=fetch + ) + monkeypatch.delenv('GITHUB_SHA') # A valid signature on a different commit cannot authorize this checkout. git('commit', '--allow-empty', '-m', 'Different workflow commit') for enforce in (False, True):