diff --git a/.changes/README.md b/.changes/README.md new file mode 100644 index 000000000..191a46de3 --- /dev/null +++ b/.changes/README.md @@ -0,0 +1,9 @@ +# Release-note sources + +Add a fragment with `go run github.com/miniscruff/changie@v1.24.0 new`. +Unreleased YAML fragments belong in `unreleased/`. + +`v2.13.5.md` preserves the complete changelog history through v2.13.5, imported +without changing any released section. Future batches create one file per version. +Keep this historical archive intact. Regenerate `CHANGELOG.md` with +`go run github.com/miniscruff/changie@v1.24.0 merge` after editing a batch or header. diff --git a/.changes/header.tpl.md b/.changes/header.tpl.md new file mode 100644 index 000000000..f94cef0c6 --- /dev/null +++ b/.changes/header.tpl.md @@ -0,0 +1,8 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Unreleased entries are in [`.changes/unreleased/`](.changes/unreleased/). +See [CONTRIBUTING.md](CONTRIBUTING.md#changelog) for Changie commands. diff --git a/changelog.d/.gitkeep b/.changes/unreleased/.gitkeep similarity index 100% rename from changelog.d/.gitkeep rename to .changes/unreleased/.gitkeep diff --git a/.changes/unreleased/release-standard.yaml b/.changes/unreleased/release-standard.yaml new file mode 100644 index 000000000..317be5c59 --- /dev/null +++ b/.changes/unreleased/release-standard.yaml @@ -0,0 +1,5 @@ +kind: Infrastructure +body: >- + Migrate release notes to Changie, use PyPI Trusted Publishing with attestations, + and add release security checks and a staged signed-tag verification gate. +time: 2026-11-17T00:00:00Z diff --git a/.changes/v2.13.5.md b/.changes/v2.13.5.md new file mode 100644 index 000000000..73d95ac29 --- /dev/null +++ b/.changes/v2.13.5.md @@ -0,0 +1,1069 @@ +## [2.13.5](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.5) - 2026-11-17 + + +### Infrastructure + +- Make the sdkharness legal-hold delete check accept B2's `401 access_denied` response while retaining older simulator refusal shapes. + + +## [2.13.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.4) - 2026-11-12 + +### Fixed + +- Notification-rule methods and B2 error interpretation no longer record request arguments (including signing secrets, custom headers, and SSE-C keys) in SDK debug tracing. + + +## [2.13.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.3) - 2026-11-06 + + +### Fixed + +- Accept `reporter=None` in `Synchronizer.sync_folders` by using a silent reporter instead of failing. +- Keep scanning a local folder after a file cannot be read or accessed when no reporter is given, instead of failing on that file. +- Parallel download: hash the data that was actually written, and surface a failing writer to producers instead of blocking them or losing the error. +- Pass `expires`, `content_disposition`, `content_encoding` and `content_language` through `Bucket.copy` to the created file instead of dropping them. +- Send `customUploadTimestamp` under its B2 wire name when starting a large file, so custom upload timestamps are no longer ignored. +- Stop the replication monitor from hanging when its consumer fails: the producer no longer blocks on a full queue and always signals the end of input. +- Store the account's allowed capabilities in SQLite with a bound parameter instead of string formatting, and reset cached upload pools when `SqliteAccountInfo.clear()` is called. + +### Infrastructure + +- Updated the locked urllib3 dependency to version 2.8.0. ([#638](https://github.com/Backblaze/b2-sdk-python/issues/638)) +- Updated Starlette, AnyIO, and virtualenv in the development tooling. Setuptools remains below 81 for liccheck compatibility and requires a dev-only dismissal; filelock and tornado retain Python 3.10-compatible dev-only locks. ([#640](https://github.com/Backblaze/b2-sdk-python/issues/640)) + + +## [2.13.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.2) - 2026-10-21 + + +### Changed + +- Require certifi 2026.10.19 or newer. + +### Infrastructure + +- Make the sdkharness `upload.stall` check runnable: it meets its request-timeout need through the documented `B2HttpApiConfig(http_session_factory=...)` instead of reporting `no-client-option`. +- Require pip 26.2.2 or newer in the Nox runner, lint, build, and test environments and update GitHub Actions to uv 0.13.4 to address the emSign CA certificate security incident. + + +## [2.13.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.1) - 2026-10-04 + +### Fixed + +- Retry uploads after a B2 request-timeout response (HTTP 408). ([#605](https://github.com/Backblaze/b2-sdk-python/issues/605)) +- Fixed honoring the Retry-After header on 5xx service errors. ([#606](https://github.com/Backblaze/b2-sdk-python/issues/606)) + +### Infrastructure + +- Added a repository-owned sdkharness customer-health contract for version-matched simulator testing. ([#610](https://github.com/Backblaze/b2-sdk-python/issues/610)) +- Moved sdkharness conformance and resilience assertions into this repository so they execute against the exact b2sdk checkout. ([#612](https://github.com/Backblaze/b2-sdk-python/issues/612)) +- Document in the sdkharness README how to run one check against the standalone simulator, and list every proxy variable the leaf checks scrub. +- Make the sdkharness leaf-check unit tests ignore `__pycache__` and other non-leaf entries. +- Repository-owned conformance checks `lock.bypass_governance` and `lock.per_file_retention` accept the 401 `access_denied` that B2 answers for a delete of a governance-locked version without bypass, and still fail if the delete is allowed or refused for an unrelated reason. +- Repository-owned sdkharness conformance and resilience checks now report an authorization error, access denied, a connection error against the simulator, or an SDK `ImportError` as `FAIL` instead of `COULD-NOT-RUN`/`SKIP`, strip proxy environment variables so a stray proxy cannot change a result, and no longer refer to files that exist only in the central harness. +- Repository-owned sdkharness conformance and resilience leaf checks now refuse to run unless they are given a loopback simulator URL, and always use the fixed simulator credential, so running one directly can no longer reach staging or production with ambient `B2_*` credentials. +- The repository-owned resilience dispatcher refuses a simulator that has already served requests, instead of letting a leaf count an earlier scenario's journal entries (for example `upload.cap_exceeded_403` reporting extra `b2_upload_file` calls) and report a false SDK failure. + + +## [2.13.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.0) - 2026-09-14 + + +### Changed + +- Follow B2's switch to SSE-B2 as the default server-side encryption: an omitted encryption setting now means the server default (SSE-B2) for both bucket defaults and file writes, and an explicit `none` is rejected before the request is sent, with `WrongEncryptionModeForBucketDefault` for bucket create/update and the new `WrongEncryptionSettingForFileWrite` for uploads and copies. `RawSimulator` applies the same rules. + +### Added + +- Add `EncryptionSetting.can_be_used_for_file_write()` and the `WrongEncryptionSettingForFileWrite` exception. + +### Infrastructure + +- Add a build-time smoke test that installs the freshly built source distribution and imports the public API shims. +- Add a checked-in Read the Docs build configuration so docs builds install the required doc dependencies and system packages. +- Bump urllib3 from 2.6.3 to 2.7.0. +- Run B2 CLI unit tests and integration tests in CI against the SDK checkout. + + +## [2.12.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.12.0) - 2026-05-05 + + +### Removed + +- Drop support for Python 3.9 and PyPy 3.9. + +### Infrastructure + +- Upgrade test dependencies versions. + + +## [2.11.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.11.0) - 2026-05-01 + + +### Fixed + +- Handle null case when encoding / decoding `content_sha1` in BaseFileVersion. ([#553](https://github.com/Backblaze/b2-sdk-python/issues/553)) +- Fix `SqliteAccountInfo` to explicitly close temporary sqlite connections used during setup, add a `close()` method for cached connection. + +### Added + +- Declare official support for Python 3.14 and include it in the CI test matrix. +- Introduce a reusable validator to ensure remote B2 file names are safe to use as local paths. + +### Infrastructure + +- Bump tornado dependency from 6.5.4 to 6.5.5. + + +## [2.10.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.4) - 2026-03-03 + + +### Fixed + +- Fixed a retry bug in `upload_unbound_stream()` small-file uploads where a retryable upload error could cause a one-shot buffered stream to be reopened after it was closed, raising `ValueError: I/O operation on closed file`. + +### Infrastructure + +- Add exponential retries for bucket creation in `testing.helpers.BucketManager`. +- Fix redudant files in the sdist after migrating from `pdm` to `uv`. + + +## [2.10.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.3) - 2026-02-23 + + +### Fixed + +- Address backwards compatibility issue for sqlite account info caused by the migration of schema to a new multi-bucket format. +- Avoid http-level retries during upload requests. +- Use `stat.S_ISDIR` check for local folder children scanning instead of `Path.is_dir` to account for an api change in Python 3.14. + +### Infrastructure + +- Bump pip version. +- Migrate from pdm to uv. +- Move pytest_plugins to top-level conftest (fix for newer pytest). +- Set temporary directory as account info config dir in tests. +- Use a separate nox dependency group for reprocible installs of nox in ci / cd. + + +## [2.10.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.2) - 2025-12-16 + + +### Fixed + +- Fix type error in retrying behavior for 429 status with Retry-After header value set. + +### Infrastructure + +- Improve debugging in integration tests bucket manager. +- Redesign `b2http` unit tests utilizing sentry `responses` library for mocking out the `requests` package. +- Update CI to not use deprecated `macos-13` runner. + + +## [2.10.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.1) - 2025-10-26 + + +### Changed + +- Improve seed generation for containers' names in tests. + + +## [2.10.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.10.0) - 2025-08-18 + + +### Removed + +- Drop Python 3.8 support. + +### Changed + +- Improved test helpers and fixtures and moved them to the public API. + + +## [2.9.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.9.4) - 2025-06-11 + + +### Fixed + +- Fix broken `StubAccountInfo.list_bucket_names_ids()`. +- Fix bucket name mapping in `RawSimulator.authorize_account()`. +- Fix incorrect path check in `LocalFolder.make_full_path`. + + +## [2.9.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.9.3) - 2025-05-29 + + +### Fixed + +- Fix `B2HttpApiConfig` and `Services` references in exposed `v2` API. + + +## [2.9.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.9.2) - 2025-05-29 + + +### Fixed + +- Fix incorrect reliance on v3 abstractions in apiver v2. ([#540](https://github.com/Backblaze/b2-sdk-python/issues/540)) + + +## [2.9.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.9.1) - 2025-05-28 + + +### Changed + +- Adapt authorize_account flow to multi-bucket keys. +- Migrate to b2 native api v4. +- Move url_for_api func to an internal classmethod in B2Api class. +- Release apiver v3 interface. `from b2sdk.v3 import ...` is now the recommended import, but previous versions are still supported. +- Update application key classes to support multiple bucket ids. +- Update create_key flow to multi-bucket keys. + +### Infrastructure + +- Migrate integration tests to apiver v3. + + +## [2.8.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.8.1) - 2025-04-24 + + +### Fixed + +- Fix TimeoutError handling in `b2http`. + +### Doc + +- Document params in FileRetentionSetting class. ([#532](https://github.com/Backblaze/b2-sdk-python/issues/532)) + + +## [2.8.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.8.0) - 2025-01-23 + + +### Changed + +- Migrate to B2 Native API v3. + +### Fixed + +- Fix continuation for started large files with no fully finished parts. +- Perform re-authentication for empty 401 responses returned for `HEAD` requests. + +### Infrastructure + +- Remove yapf in favor of ruff. + + +## [2.7.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.7.0) - 2024-12-12 + + +### Changed + +- Make Event Notifications generally available. ([#518](https://github.com/Backblaze/b2-sdk-python/issues/518)) +- Switch a pytest hook from path to collection_path. + +### Fixed + +- Add upload token reset after upload timeout. +- Fix file/directory permission handling for Windows during the B2 sync. + +### Infrastructure + +- Fix event notification tests when introducing new keys in API outputs. + + +## [2.6.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.6.0) - 2024-10-28 + + +### Removed + +- Remove Python 3.7 support in new releases. + Under Python 3.7 `pip` will keep resolving the latest version of the package that supports active interpreter. + Python 3.8 is now the minimum supported version, [until it reaches EOL in October 2024](https://devguide.python.org/versions/). + We encourage use of the latest stable Python release. + +### Fixed + +- Fixed datetime.utcnow() deprecation warnings under Python 3.12. + +### Added + +- Declare official support for Python 3.13 in b2sdk. + Test b2sdk against Python 3.13 in CI. + +### Infrastructure + +- Upgraded to pytest 8 (#484). + + +## [2.5.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.5.1) - 2024-08-15 + + +### Fixed + +- Fix LocalFolder.all_files(..) erroring out if one of the non-excluded directories is not readable by the user running the scan. + Warning is added to ProgressReport instead as other file access errors are. + + +## [2.5.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.5.0) - 2024-07-30 + + +### Fixed + +- Fix TruncatedOutput errors when downloading files over congested network (fixes [B2_Command_Line_Tool#554](https://github.com/Backblaze/B2_Command_Line_Tool/issues/554)). +- Ensure `FileSimulator.as_download_headers` returns `dict[str, str]` mapping. + +### Added + +- Add `unhide_file` method to Bucket class. + +### Doc + +- Improve `download_file_from_url` methods type hints. + +### Infrastructure + +- Limit max CI (Github Actions) duration to 90 minutes. + + +## [2.4.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.4.1) - 2024-06-19 + + +### Fixed + +- Fix `LocalFolder` regression (introduced in 2.4.0) which caused `LocalFolder` to not list files by path lexicographical order. + This is also a fix for `synchronizer` re-uploading files on every run in some cases. ([#502](https://github.com/Backblaze/b2-sdk-python/issues/502)) + + +## [2.4.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.4.0) - 2024-06-17 + + +### Changed + +- In `b2sdk.v3` the `B2Api` will always create `cache` from `AccountInfo` object, unless `cache` is provided explicitly. + The current stable `b2sdk.v2` remains unchanged, i.e. `DummyCache` is created by default if `account_info` was provided, but not `cache`. + Documentation for `b2sdk.v2` was updated with the new recommended usage, e.g. `B2Api(info, cache=AuthInfoCache(info))`, to achieve the same behavior as `b2sdk.v3`. ([#497](https://github.com/Backblaze/b2-sdk-python/issues/497)) + +### Fixed + +- Move scan filters before a read on filesystem access attempt. This will prevent unnecessary warnings and IO operations on paths that are not relevant to the operation. ([#456](https://github.com/Backblaze/b2-sdk-python/issues/456)) +- Fix bucket caching erroring out when using `StubAccountInfo`. + +### Added + +- Add `annotated_types` dependency for type annotations that include basic value validation. +- Add `daysFromStartingToCancelingUnfinishedLargeFiles` option to `lifecycle_rules` type annotation. +- Add non-retryable `NoPaymentHistory` exception. + API returns this exception when action (e.g. bucket creation or replication rules) is not allowed due to lack of payment history. + + +## [2.3.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.3.0) - 2024-05-15 + + +### Added + +- Add `folder_to_list_can_be_a_file` parameter to `b2sdk.v2.Bucket.ls`, that if set to `True` will allow listing a file versions if path is an exact match. + This parameter won't be included in `b2sdk.v3.Bucket.ls` and unless supplied `path` ends with `/`, the possibility of path pointing to file will be considered first. + + +## [2.2.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.2.1) - 2024-05-09 + + +### Fixed + +- Fix `__str__` of `b2sdk.v2.BucketIdNotFound` to return full error message and not just missing bucket ID value. + + +## [2.2.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.2.0) - 2024-05-08 + +### Added + +- Add `has_errors_or_warnings` method to `ProgressReport` class. + +### Fixed + +- Ensure `b2sdk.v2.b2http` emits `b2sdk.v2.BucketIdNotFound` exception instead of `b2sdk._v3.BucketIdNotFound`. ([#437](https://github.com/Backblaze/b2-sdk-python/issues/437)) +- Ensure `unprintable_to_hex` and `unprintable_to_hex` return empty string (instead of `None`) if empty string was supplied as argument. +- Skip files with invalid filenames when scanning directories (for `sync`, ...) instead of raising an exception. + + +## [2.1.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.1.0) - 2024-04-15 + + +### Changed + +- Use ParallelDownloader for small files instead of SimpleDownloader to avoid blocking on I/O. + +### Fixed + +- Fix `decode_content=True` causing an error when downloading tiny and large files. +- Prevent errors due to the use of "seekable" download strategies for seekable, but not readable files. + +### Added + +- Add set&get Event Notification rules methods to Bucket API as part of Event Notifications feature Private Preview. + See https://www.backblaze.com/blog/announcing-event-notifications/ for details. + + +## [2.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.0.0) - 2024-04-02 + + +### Removed + +- Remove `tqdm` dependency. Now `tqdm` has to be explicitly installed to use `TqdmProgressListener` class. +- Remove `[doc]` extras dependency group - moved to dev dependencies. +- Remove unnecessary `packaging` package dependency. It's functionality was never explicitly exposed. + +### Changed + +- Move non-apiver packages (e.g. packages other than `b2sdk.v1`, `b2sdk.v2`, ...) to `b2sdk._internal` to further discourage use of non-public internals. + If you accidentally used non-public internals, most likely only thing you will need to do, is import from `b2sdk.v2` instead of `b2sdk`. +- Move logging setup and `UrllibWarningFilter` class from `b2sdk.__init__.py` to `b2sdk._v3` (and thus `b2sdk.v2` & `b2sdk.v1`). + This will allow us to remove/change it in new apiver releases without the need to change the major semver version. + +### Added + +- Add `SqliteAccountInfo.get_user_account_info_path` to public API. + +### Infrastructure + +- Update to [GitHub Actions using Node 20](https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/). + + +## [1.33.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.33.0) - 2024-03-15 + + +### Fixed + +- Escape control characters whenever printing object and bucket names to improve security. +- Remove unused `setuptools` from default dependency list. + +### Added + +- Added control character escaping methods. + + +## [1.32.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.32.0) - 2024-02-26 + + +### Added + +- Add `set_thread_pool_size`, `get_thread_pool_size` to *Manger classes. + +### Infrastructure + +- Fix schema graph rendering in readthedocs documentation. + + +## [1.31.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.31.0) - 2024-02-19 + + +### Fixed + +- Remove obsolete test scripts from b2sdk package: `test_upload_url_concurrency`, `b2sdk.b2http:test_http`. ([#471](https://github.com/Backblaze/b2-sdk-python/issues/471)) + +### Added + +- Allow for `min_part_size` that is greater than default `recommended_part_size` value, without having to explicitly set `recommended_part_size` value. +- Add `GET` method support to `B2Http`. +- Add `JSON` type annotation and fix type hints in `B2Http` methods. +- Add more type hints to API methods. + + +## [1.30.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.30.1) - 2024-02-02 + + +### Fixed + +- Fix package author metadata. + + +## [1.30.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.30.0) - 2024-02-02 + + +### Fixed + +- Fix escape sequence warnings present in python 3.12. ([#458](https://github.com/Backblaze/b2-sdk-python/issues/458)) +- Handle json encoded, invalid B2 error responses, preventing exceptions such as `invalid literal for int() with base 10: 'service_unavailable'`. + +### Added + +- Add support for filters to `Bucket.ls()`. + +### Infrastructure + +- Package the library using [pdm](https://pdm-project.org), use locked dependencies in CI. +- Update `ruff` linter and apply it to all files. + + +## [1.29.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.29.1) - 2024-01-23 + + +### Fixed + +- Handle non-json encoded B2 error responses, i.e. retry on 502 and 504 errors. + +### Doc + +- Add missing import in Synchronizer docs example. + + +## [1.29.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.29.0) - 2023-12-13 + + +### Changed + +- Change v3.B2Api.authorize_account signature to make `realm` optional and `"production"` by default. + +### Added + +- Progress listener instances can now change their descriptions during run. This allows for e.g.: changing description after file headers are downloaded but before the content is fetched. + +### Infrastructure + +- Add `-v` to pytest in CI. +- Run windows pypy3.9 tests on nightly builds. + + +## [1.28.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.28.0) - 2023-12-06 + + +### Changed + +- On XDG compatible OSes (Linux, BSD), the profile file is now created in `$XDG_CONFIG_HOME` (with a fallback to `~/.config/` in absence of given env. variable). +- Replace blank `assert` with exception when size values for parts upload are misaligned. + +### Fixed + +- Streaming from empty stream no longer ends with "Empty emerge parts iterator" error. + +### Infrastructure + +- Changelog entries are now validated as a part of CI pipeline. +- Disable dependabot requests for updates unrelated to security issues. +- Fixed tests failing because of changes made to `locale.normalize` in Python 3.12. + + +## [1.27.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.27.0) - 2023-11-26 + + +### Changed + +- Add dependency on `setuptools` and `packaging` as they are not shipped by cpython 3.12 and are used in production code. + +### Fixed + +- Fix closing of passed progress listeners in `Bucket.upload` and `Bucket.copy` + + +## [1.26.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.26.0) - 2023-11-20 + + +### Added + +- Add `expires`, `content_disposition`, `content_encoding`, `content_language` arguments to various `Bucket` methods ([#357](https://github.com/Backblaze/b2-sdk-python/issues/357)) + +### Infrastructure + +- Towncrier changelog generation - to avoid conflicts when simultaneously working on PRs +- Fix towncrier generated changelog to work with mindsers/changelog-reader-action + + +## [1.25.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.25.0) - 2023-11-15 + +### Added +- Add `*_PART_SIZE`, `BUCKET_NAME_*`, `STDOUT_FILEPATH` constants +- Add `points_to_fifo`, `points_to_stdout` functions + +### Changed +- Mark `TempDir` as deprecated in favor of `tempfile.TemporaryDirectory` + +### Fixed +- Fix downloading to a non-seekable file, such as /dev/stdout +- Fix ScanPoliciesManager support for compiled regexes + +### Infrastructure +- Fix readthedocs build by updating to v2 configuration schema +- Fix spellcheck erroring out on LICENSE file +- Fix snyk reporting vulnerability due to tornado package use in docs generation +- Deduplicate test_base files in test suite +- Refactor integration tests for better pytest compatibility & eager bucket cleanup + +## [1.24.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.24.1) - 2023-09-27 + +### Fixed +- Fix missing key ID for large file encrypted with SSE-C +- Fix concatenating error message when message is None + +## [1.24.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.24.0) - 2023-08-31 + +### Added +- 'bypass_governance' flag to delete_file_version + +## [1.23.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.23.0) - 2023-08-10 + +### Added +- Add `get_file_info_by_name` to the B2Api class + +### Fixed +- Require `typing_extensions` on Python 3.11 (already required on earlier versions) for better compatibility with pydantic v2 +- Fix `RawSimulator` handling of `cache_control` parameter during tests. + +## [1.22.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.22.1) - 2023-07-24 + +### Fixed +- Fix regression in dir exclusion patterns introduced in 1.22.0 + +## [1.22.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.22.0) - 2023-07-21 + +### Added +- Declare official support of Python 3.12 +- Improved `lifecycle_rules` argument type annotations + +### Deprecated +- Deprecate `file_infos` argument. Use `file_info` instead. Old argument name won't be supported in v3. + +### Changed +- `version_utils` decorators now ignore `current_version` parameter to better fit `apiver` needs + +### Fixed +- Circular symlinks no longer cause infinite loops when syncing a folder +- Fix crash on upload retry with unbound data source + +### Infrastructure +- Remove unsupported PyPy versions (3.7, 3.8) from tests matrix and add PyPy 3.9 & 3.10 instead +- Replaced `pyflakes` with `ruff` for linting +- Refactored logic for resuming large file uploads to unify code paths, correct inconsistencies, and enhance configurability (#381) +- Automatically set copyright date when generating the docs +- Use modern type hints in documentation (achieved through combination of PEP 563 & 585 and `sphinx-autodoc-typehints`) + +## [1.21.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.21.0) - 2023-04-17 + +### Added +- Add support for custom upload timestamp +- Add support for cache control header while uploading + +### Infrastructure +- Remove dependency from `arrow` +- Build Python wheels for distribution + +## [1.20.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.20.0) - 2023-03-23 + +### Added +- Add `use_cache` parameter to `B2Api.list_buckets` + +### Changed +- Connection timeout is now being set explicitly + +### Fixed +- Small files downloaded twice + +### Infrastructure +- Disable changelog verification for dependabot PRs + +## [1.19.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.19.0) - 2023-01-24 + +### Added +- Authorizing a key for a single bucket ensures that this bucket is cached +- `Bucket.ls` operation supports wildcard matching strings +- Documentation for `AbstractUploadSource` and its children +- `InvalidJsonResponse` when the received error is not a proper JSON document +- Raising `PotentialS3EndpointPassedAsRealm` when a specific misconfiguration is suspected +- Add `large_file_sha1` support +- Add support for incremental upload and sync +- Ability to stream data from an unbound source to B2 (for example stdin) + +### Fixed +- Removed information about replication being in closed beta +- Don't throw raw `OSError` exceptions when using `DownloadedFile.save_to` to a path that doesn't exist, is a directory or the user doesn't have permissions to write to + +### Infrastructure +- Additional tests for listing files/versions +- Ensured that changelog validation only happens on pull requests +- Upgraded GitHub actions checkout to v3, python-setup to v4 +- Additional tests for `IncrementalHexDigester` + +## [1.18.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.18.0) - 2022-09-20 + +### Added +- Logging performance summary of parallel download threads +- Add `max_download_streams_per_file` parameter to B2Api class and underlying structures +- Add `is_file_lock_enabled` parameter to `Bucket.update()` and related methods + +### Fixed +- Replace `ReplicationScanResult.source_has_sse_c_enabled` with `source_encryption_mode` +- Fix `B2Api.get_key()` and `RawSimulator.delete_key()` +- Fix calling `CopySizeTooBig` exception + +### Infrastructure +- Fix nox's deprecated `session.install()` calls +- Re-enable changelog validation in CI +- StatsCollector contains context managers for gathering performance statistics + +## [1.17.3](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.3) - 2022-07-15 + +### Fixed +- Fix `FileVersion._get_upload_headers` when encryption key is `None` + +### Infrastructure +- Fix download integration tests on non-production environments +- Add `B2_DEBUG_HTTP` env variable to enable network-level test debugging +- Disable changelog validation temporarily + +## [1.17.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.2) - 2022-06-24 + +### Fixed +- Fix a race in progress reporter +- Fix import of replication + +## [1.17.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.1) - 2022-06-23 [YANKED] + +### Fixed +- Fix importing scan module + +## [1.17.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.17.0) - 2022-06-23 [YANKED] + +As in version 1.16.0, the replication API may still be unstable, however +no backward-incompatible changes are planned at this point. + +### Added +- Add `included_sources` module for keeping track of included modified third-party libraries +- Add `include_existing_files` parameter to `ReplicationSetupHelper` +- Add `get_b2sdk_doc_urls` function for extraction of external documentation URLs during runtime + +### Changed +- Downloading compressed files with `Content-Encoding` header set no longer causes them to be decompressed on the fly - it's an option +- Change the per part retry limit from 5 to 20 for data transfer operations. Please note that the retry system is not considered to be a part of the public interface and is subject to be adjusted +- Do not wait more than 64 seconds between retry attempts (unless server asks for it) +- On longer failures wait an additional (random, up to 1s) amount of time to prevent client synchronization +- Flatten `ReplicationConfiguration` interface +- Reorder actions of `ReplicationSetupHelper` to avoid zombie rules + +### Fixed +- Fix: downloading compressed files and decompressing them on the fly now does not cause a TruncatedOutput error +- Fix `AccountInfo.is_master_key()` +- Fix docstring of `SqliteAccountInfo` +- Fix lifecycle rule type in the docs + +### Infrastructure +- Add 3.11.0-beta.1 to CI +- Change Sphinx major version from 5 to 6 +- Extract folder/bucket scanning into a new `scan` module +- Enable pip cache in CI + +## [1.16.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.16.0) - 2022-04-27 + +This release contains a preview of replication support. It allows for basic +usage of B2 replication feature (currently in closed beta). + +As the interface of the sdk (and the server api) may change, the replication +support shall be considered PRIVATE interface and should be used with caution. +Please consult the documentation on how to safely use the private api interface. + +Expect substantial amount of work on sdk interface: +- The interface of `ReplicationConfiguration` WILL change +- The interface of `FileVersion.replication_status` MIGHT change +- The interface of `FileVersionDownload` MIGHT change + +### Added +- Add basic replication support to `Bucket` and `FileVersion` +- Add `is_master_key()` method to `AbstractAccountInfo` +- Add `readBucketReplications` and `writeBucketReplications` to `ALL_CAPABILITIES` +- Add log tracing of `interpret_b2_error` +- Add `ReplicationSetupHelper` + +### Fixed +- Fix license test on Windows +- Fix cryptic errors when running integration tests with a non-full key + +## [1.15.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.15.0) - 2022-04-12 + +### Changed +- Don't run coverage in pypy in CI +- Introduce a common thread worker pool for all downloads +- Increase http timeout to 20min (for copy using 5GB parts) +- Remove inheritance from object (leftover from python2) +- Run unit tests on all CPUs + +### Added +- Add pypy-3.8 to test matrix +- Add support for unverified checksum upload mode +- Add dedicated exception for unverified email +- Add a parameter to customize `sync_policy_manager` +- Add parameters to set the min/max part size for large file upload/copy methods +- Add CopySourceTooBig exception +- Add an option to set a custom file version class to `FileVersionFactory` +- Add an option for B2Api to turn off hash checking for downloaded files +- Add an option for B2Api to set write buffer size for `DownloadedFile.save_to` method +- Add support for multiple profile files for SqliteAccountInfo + +### Fixed +- Fix copying objects larger than 1TB +- Fix uploading objects larger than 1TB +- Fix downloading files with unverified checksum +- Fix decoding in filename and file info of `DownloadVersion` +- Fix an off-by-one bug and other bugs in the Simulator copy functionality + +### Removed +- Drop support for Python 3.5 and Python 3.6 + +## [1.14.1](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.14.1) - 2022-02-23 + +### Security +- Fix setting permissions for local sqlite database (thanks to Jan Schejbal for responsible disclosure!) + +## [1.14.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.14.0) - 2021-12-23 + +### Fixed +- Relax constraint on arrow to allow for versions >= 1.0.2 + +## [1.13.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.13.0) - 2021-10-24 + +### Added +- Add support for Python 3.10 + +### Changed +- Update a list with all capabilities + +### Fixed +- Fix pypy selector in CI + +## [1.12.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.12.0) - 2021-08-06 + +### Changed +- The `importlib-metadata` requirement is less strictly bound now (just >=3.3.0 for python > 3.5). +- `B2Api` `update_file_legal_hold` and `update_file_retention_setting` now return the set values + +### Added +- `BucketIdNotFound` thrown based on B2 cloud response +- `_clone` method to `FileVersion` and `DownloadVersion` +- `delete`, `update_legal_hold`, `update_retention` and `download` methods added to `FileVersion` + +### Fixed +- FileSimulator returns special file info headers properly + +### Removed +- One unused import. + +## [1.11.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.11.0) - 2021-06-24 + +### Changed +- apiver `v2` interface released. `from b2sdk.v2 import ...` is now the recommended import, + but `from b2sdk.v1 import ...` works as before + +## [1.10.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.10.0) - 2021-06-23 + +### Added +- `get_fresh_state` method added to `FileVersion` and `Bucket` + +### Changed +- `download_file_*` methods refactored to allow for inspecting DownloadVersion before downloading the whole file +- `B2Api.get_file_info` returns a `FileVersion` object in v2 +- `B2RawApi` renamed to `B2RawHTTPApi` +- `B2HTTP` tests are now common +- `B2HttpApiConfig` class introduced to provide parameters like `user_agent_append` to `B2Api` without using internal classes in v2 +- `Bucket.update` returns a `Bucket` object in v2 +- `Bucket.ls` argument `show_versions` renamed to `latest_only` in v2 +- `B2Api` application key methods refactored to operate with dataclasses instead of dicts in v2 +- `B2Api.list_keys` is a generator lazily fetching all keys in v2 +- `account_id` and `bucket_id` added to FileVersion + +### Fixed +- Fix EncryptionSetting.from_response_headers +- Fix FileVersion.size and FileVersion.mod_time_millis type ambiguity +- Old buckets (from past tests) are cleaned up before running integration tests in a single thread + +### Removed +- Remove deprecated `SyncReport` methods + +## [1.9.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.9.0) - 2021-06-07 + +### Added +- `ScanPoliciesManager` is able to filter b2 files by upload timestamp + +### Changed +- `Synchronizer.make_file_sync_actions` and `Synchronizer.make_folder_sync_actions` were made private in v2 interface +- Refactored `sync.file.*File` and `sync.file.*FileVersion` to `sync.path.*SyncPath` in v2 +- Refactored `FileVersionInfo` to `FileVersion` in v2 +- `ScanPoliciesManager` exclusion interface changed in v2 +- `B2Api` unittests for v0, v1 and v2 are now common +- `B2Api.cancel_large_file` returns a `FileIdAndName` object instead of a `FileVersion` object in v2 +- `FileVersion` has a mandatory `api` parameter in v2 +- `B2Folder` holds a handle to B2Api +- `Bucket` unit tests for v1 and v2 are now common + +### Fixed +- Fix call to incorrect internal api in `B2Api.get_download_url_for_file_name` + +## [1.8.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.8.0) - 2021-05-21 + +### Added +- Add `get_bucket_name_or_none_from_bucket_id` to `AccountInfo` and `Cache` +- Add possibility to change realm during integration tests +- Add support for "file locks": file retention, legal hold and default bucket retention + +### Fixed +- Cleanup sync errors related to directories +- Use proper error handling in `ScanPoliciesManager` +- Application key restriction message reverted to previous form +- Added missing apiver wrappers for FileVersionInfo +- Fix crash when Content-Range header is missing +- Pin dependency versions appropriately + +### Changed +- `b2sdk.v1.sync` refactored to reflect `b2sdk.sync` structure +- Make `B2Api.get_bucket_by_id` return populated bucket objects in v2 +- Add proper support of `recommended_part_size` and `absolute_minimum_part_size` in `AccountInfo` +- Refactored `minimum_part_size` to `recommended_part_size` (the value used stays the same) +- Encryption settings, types and providers are now part of the public API + +### Removed +- Remove `Bucket.copy_file` and `Bucket.start_large_file` +- Remove `FileVersionInfo.format_ls_entry` and `FileVersionInfo.format_folder_ls_entry` + +## [1.7.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.7.0) - 2021-04-22 + +### Added +- Add `__slots__` and `__eq__` to `FileVersionInfo` for memory usage optimization and ease of testing +- Add support for SSE-C server-side encryption mode +- Add support for `XDG_CONFIG_HOME` for determining the location of `SqliteAccountInfo` db file + +### Changed +- `BasicSyncEncryptionSettingsProvider` supports different settings sets for reading and writing +- Refactored AccountInfo tests to a single file using pytest + +### Fixed +- Fix clearing cache during `authorize_account` +- Fix `ChainedStream` (needed in `Bucket.create_file` etc.) +- Make tqdm-based progress reporters less jumpy and easier to read +- Fix emerger examples in docs + +## [1.6.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.6.0) - 2021-04-08 + +### Added +- Fetch S3-compatible API URL from `authorize_account` + +### Fixed +- Exclude packages inside the test package when installing +- Fix for server response change regarding SSE + +## [1.5.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.5.0) - 2021-03-25 + +### Added +- Add `dependabot.yml` +- Add support for SSE-B2 server-side encryption mode + +### Changed +- Add upper version limit for the requirements + +### Fixed +- Pin `setuptools-scm<6.0` as `>=6.0` doesn't support Python 3.5 + +## [1.4.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.4.0) - 2021-03-03 + +### Changed +- Add an ability to provide `bucket_id` filter parameter for `list_buckets` +- Add `is_same_key` method to `AccountInfo` +- Add upper version limit for arrow dependency, because of a breaking change + +### Fixed +- Fix docs autogen + +## [1.3.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.3.0) - 2021-01-13 + +### Added +- Add custom exception for `403 transaction_cap_exceeded` +- Add `get_file_info_by_id` and `get_file_info_by_name` to `Bucket` +- `FileNotPresent` and `NonExistentBucket` now subclass new exceptions `FileOrBucketNotFound` and `ResourceNotFound` + +### Changed +- Fix missing import in the synchronization example +- Use `setuptools-scm` for versioning +- Clean up CI steps + +## [1.2.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.2.0) - 2020-11-03 + +### Added +- Add support for Python 3.9 +- Support for bucket to bucket sync +- Add a possibility to append a string to the User-Agent in `B2Http` + +### Changed +- Change default fetch count for `ls` to 10000 + +### Removed +- Drop Python 2 and Python 3.4 support :tada: +- Remove `--prefix` from `ls` (it didn't really work, use `folderName` argument) + +### Fixed +- Allow to set an empty bucket info during the update +- Fix docs generation in CI + +## [1.1.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.4) - 2020-07-15 + +### Added +- Allow specifying custom realm in B2Session.authorize_account + +## [1.1.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.2) - 2020-07-06 + +### Fixed +- Fix upload part for file range on Python 2.7 + +## [1.1.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.1.0) - 2020-06-24 + +### Added +- Add `list_file_versions` method to buckets. +- Add server-side copy support for large files +- Add ability to synthesize objects from local and remote sources +- Add AuthInfoCache, InMemoryCache and AbstractCache to public interface +- Add ability to filter in ScanPoliciesManager based on modification time +- Add ScanPoliciesManager and SyncReport to public interface +- Add md5 checksum to FileVersionInfo +- Add more keys to dicts returned by as_dict() methods + +### Changed +- Make sync treat hidden files as deleted +- Ignore urllib3 "connection pool is full" warning + +### Removed +- Remove arrow warnings caused by https://github.com/crsmithdev/arrow/issues/612 + +### Fixed +- Fix handling of modification time of files + +## [1.0.2](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.2) - 2019-10-15 + +### Changed +- Remove upper version limit for arrow dependency + +## [1.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.0) - 2019-10-03 + +### Fixed +- Minor bug fix. + +## [1.0.0](https://github.com/Backblaze/b2-sdk-python/releases/tag/v1.0.0)-rc1 - 2019-07-09 + +### Deprecated +- Deprecate some transitional method names to v0 in preparation for v1.0.0. + +## [0.1.10](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.10) - 2019-07-09 + +### Removed +- Remove a parameter (which did nothing, really) from `b2sdk.v1.Bucket.copy_file` signature + +## [0.1.8](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.8) - 2019-06-28 + +### Added +- Add support for b2_copy_file +- Add support for `prefix` parameter on ls-like calls + +## [0.1.6](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.6) - 2019-04-24 + +### Changed +- Rename account ID for authentication to application key ID. +Account ID is still backwards compatible, only the terminology +has changed. + +### Fixed +- Fix transferer crashing on empty file download attempt + +## [0.1.4](https://github.com/Backblaze/b2-sdk-python/releases/tag/v0.1.4) - 2019-04-04 + +### Added +Initial official release of SDK as a separate package (until now it was a part of B2 CLI) diff --git a/.changie.yaml b/.changie.yaml new file mode 100644 index 000000000..cb908b39e --- /dev/null +++ b/.changie.yaml @@ -0,0 +1,32 @@ +changesDir: .changes +unreleasedDir: unreleased +headerPath: header.tpl.md +changelogPath: CHANGELOG.md +versionExt: md +versionFormat: '## [{{.VersionNoPrefix}}](https://github.com/Backblaze/b2-sdk-python/releases/tag/v{{.VersionNoPrefix}}) - {{.Time.Format "2006-01-02"}}' +kindFormat: '### {{.Kind}}' +changeFormat: '- {{.Body}}' +kinds: + - label: Removed + auto: major + - label: Changed + auto: minor + - label: Fixed + auto: patch + - label: Deprecated + auto: minor + - label: Added + auto: minor + - label: Doc + auto: patch + - label: Infrastructure + auto: patch + - label: Security + auto: patch +newlines: + afterChangelogHeader: 0 + beforeChangelogVersion: 1 + endOfVersion: 1 + afterKind: 1 + beforeKind: 1 +envPrefix: CHANGIE_ diff --git a/.github/signing-maintainers b/.github/signing-maintainers new file mode 100644 index 000000000..b365c37e0 --- /dev/null +++ b/.github/signing-maintainers @@ -0,0 +1,4 @@ +# GitHub accounts authorized for tag signing after key enrollment. +goanpeca +ffumero2003 +sophiecarreras diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 8c0c898f2..1669d59a0 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -2,59 +2,167 @@ name: Continuous Delivery on: push: - tags: ['v*'] # push events to matching v*, i.e. v1.0, v20.15.10 + tags: ['v*'] + workflow_dispatch: + inputs: + dry_run: + description: 'Build and check release preparation without publishing anywhere' + type: boolean + required: true + default: true + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false env: - PYTHON_DEFAULT_VERSION: "3.14" - UV_VERSION: "0.13.4" + PYTHON_DEFAULT_VERSION: '3.14' + UV_VERSION: '0.13.4' jobs: - deploy: - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - B2_PYPI_PASSWORD: ${{ secrets.B2_PYPI_PASSWORD }} + release-gates: runs-on: ubuntu-latest + timeout-minutes: 10 + environment: release-security + permissions: + contents: read + actions: read + security-events: read steps: - - name: Determine if pre-release - id: prerelease_check - run: | - export IS_PRERELEASE=$([[ ${{ github.ref }} =~ [^0-9]$ ]] && echo true || echo false) - echo "prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} - uses: actions/setup-python@v6 + persist-credentials: false + - name: Require green master and a release commit on master history + env: + GH_TOKEN: ${{ github.token }} + INPUT_DRY_RUN: ${{ inputs.dry_run }} + run: python scripts/release_preflight.py + - name: Restore annotated release tag + if: github.event_name == 'push' + env: + TAG: ${{ github.ref_name }} + run: git fetch --force --no-tags origin "refs/tags/$TAG:refs/tags/$TAG" + - name: Verify maintainer tag signature + if: github.event_name == 'push' + env: + ENFORCE_SIGNED_TAGS: ${{ vars.ENFORCE_SIGNED_TAGS }} + TAG: ${{ github.ref_name }} + run: python scripts/verify_release_tag.py "$TAG" + - name: Block unresolved release security findings + env: + GH_TOKEN: ${{ secrets.RELEASE_SECURITY_TOKEN || github.token }} + REPOSITORY: ${{ github.repository }} + run: python scripts/release_security.py "$REPOSITORY" + + build: + needs: release-gates + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + version: ${{ steps.notes.outputs.version }} + prerelease: ${{ steps.notes.outputs.prerelease }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} - enable-cache: true - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Install dependencies - run: uv sync --only-group nox --locked - - name: Build the distribution - id: build + - run: uv sync --only-group nox --locked + - name: Assemble a disposable release preview + if: github.event_name == 'workflow_dispatch' + run: | + go run github.com/miniscruff/changie@v1.24.0 batch patch + go run github.com/miniscruff/changie@v1.24.0 merge + - name: Validate Changie sources + run: uv run nox -vs changelog_check + - name: Extract the exact release notes + id: notes + env: + EVENT_NAME: ${{ github.event_name }} + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = workflow_dispatch ]; then + version=$(go run github.com/miniscruff/changie@v1.24.0 latest) + version=${version#v} + else + version=${TAG#v} + test ! -n "$(find .changes/unreleased -type f ! -name .gitkeep -print -quit)" + latest=$(go run github.com/miniscruff/changie@v1.24.0 latest) + test "${latest#v}" = "$version" + fi + awk -v version="$version" -f scripts/release-notes.awk CHANGELOG.md > release-notes.md + echo "version=$version" >> "$GITHUB_OUTPUT" + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo 'prerelease=false' >> "$GITHUB_OUTPUT" + else + echo 'prerelease=true' >> "$GITHUB_OUTPUT" + fi + - name: Build and smoke-test the installed distribution run: uv run nox -vs build - - name: Read the Changelog - id: read-changelog - uses: mindsers/changelog-reader-action@v2 - with: - version: ${{ steps.build.outputs.version }} - - name: Create GitHub release and upload the distribution - id: create-release - uses: softprops/action-gh-release@v2 - with: - name: ${{ steps.build.outputs.version }} - body: ${{ steps.read-changelog.outputs.changes }} - draft: ${{ env.ACTIONS_STEP_DEBUG == 'true' }} - prerelease: ${{ steps.prerelease_check.outputs.prerelease }} - files: ${{ steps.build.outputs.asset_path }} - - name: Upload the distribution to PyPI - if: ${{ env.B2_PYPI_PASSWORD != '' && steps.prerelease_check.outputs.prerelease == 'false' }} - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.B2_PYPI_PASSWORD }} + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-distributions + path: | + dist/* + release-notes.md + if-no-files-found: error + retention-days: 7 + + publish-pypi: + needs: build + if: github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: pypi + url: https://pypi.org/p/b2sdk + permissions: + contents: read + id-token: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-distributions + - name: Publish verified distributions with PEP 740 attestations + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + packages-dir: dist/ + attestations: true + + github-release: + needs: [build, publish-pypi] + if: >- + always() && github.event_name == 'push' && needs.build.result == 'success' && + needs.publish-pypi.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-distributions + - name: Create or update the GitHub release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + TAG: ${{ github.ref_name }} + PRERELEASE: ${{ needs.build.outputs.prerelease }} + run: | + set -euo pipefail + if gh release view "$TAG" >/dev/null 2>&1; then + gh release edit "$TAG" --title "$TAG" --notes-file release-notes.md --prerelease="$PRERELEASE" + gh release upload "$TAG" ./dist/* --clobber + else + gh release create "$TAG" --verify-tag --title "$TAG" --notes-file release-notes.md --prerelease="$PRERELEASE" ./dist/* + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b58b6acf..c8a1d2d9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,22 +10,22 @@ env: PYTHON_DEFAULT_VERSION: "3.14" UV_VERSION: "0.13.4" +permissions: + contents: read + jobs: lint: timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - - uses: codespell-project/actions-codespell@2391250ab05295bddd51e36a8c6295edb6343b0e - with: - ignore_words_list: datas re-use - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -33,35 +33,49 @@ jobs: run: uv sync --only-group nox --locked - name: Run linters run: uv run nox -vs lint - - name: Validate new changelog entries - if: (contains(github.event.pull_request.labels.*.name, '-changelog') == false) && (github.event.pull_request.base.ref != '') + - name: Check spelling + # Check fragment sources; preserve immutable history and the generated changelog. + # caf is intentional escaped Unicode test data. run: | - mapfile -t changed_files < <(git diff --diff-filter=ACMR --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD) - - if [ "${#changed_files[@]}" -eq 1 ] && [ "${changed_files[0]}" = "uv.lock" ]; then - echo "Skipping changelog validation for lockfile-only change." - exit 0 - fi - - if [ -z "$(git diff --diff-filter=A --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD -- changelog.d)" ]; then - echo "No changelog item added." - exit 1 - fi - - name: Changelog validation - run: uv run nox -vs towncrier_check + uvx codespell==2.4.1 --ignore-words-list=datas,re-use,caf --skip='.git,.nox,.venv,build,dist,uv.lock,CHANGELOG.md' + uvx codespell==2.4.1 --check-hidden .changes --skip='.changes/v2.13.5.md' + changelog-check: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: ${{ env.UV_VERSION }} + - run: uv sync --only-group nox --locked + - name: Validate fragments and generated changelog + env: + EVENT_NAME: ${{ github.event_name }} + ALLOW_MISSING: ${{ contains(github.event.pull_request.labels.*.name, '-changelog') }} + run: | + args=() + if [ "$EVENT_NAME" = pull_request ]; then args+=(--base origin/master); fi + if [ "$ALLOW_MISSING" = true ]; then args+=(--allow-missing-fragment); fi + uv run nox -vs changelog_check -- "${args[@]}" build: timeout-minutes: 30 - needs: lint + needs: [lint, changelog-check] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -71,22 +85,22 @@ jobs: run: uv run nox -vs build cleanup_buckets: timeout-minutes: 30 - needs: lint + needs: [lint, changelog-check] env: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead with: version: ${{ env.UV_VERSION }} @@ -121,14 +135,14 @@ jobs: extras: "full" os: "ubuntu-latest" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -160,19 +174,19 @@ jobs: - os: windows-latest python-version: "3.12" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 path: b2-sdk-python - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: repository: Backblaze/B2_Command_Line_Tool path: B2_Command_Line_Tool - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -210,14 +224,14 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: version: ${{ env.UV_VERSION }} enable-cache: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 9518f6bd5..a2b61f642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the changes for the -upcoming release can be found in [changelog.d](changelog.d). - - +Unreleased entries are in [`.changes/unreleased/`](.changes/unreleased/). +See [CONTRIBUTING.md](CONTRIBUTING.md#changelog) for Changie commands. ## [2.13.5](https://github.com/Backblaze/b2-sdk-python/releases/tag/v2.13.5) - 2026-11-17 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c561287bc..86dbbaa47 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,32 +26,26 @@ established by reading git tags, i.e. no code or manifest file changes are requi ## Changelog -Each PR needs to have at least one changelog (aka news) item added. This is done by creating files in `changelog.d`. -`towncrier` is used for compiling these files into [CHANGELOG.md](CHANGELOG.md). There are several types of changes -(news): - -1. fixed -2. changed -3. added -4. deprecated -5. removed -6. infrastructure -7. doc - - -The `changelog.d` file name convention is: - -1. If the PR closes a github issue: `{issue_number}.{type}.md` e.g. `157.fixed.md`. Note that the - change description still has to be complete, linking an issue is just there for convenience, a change like - `fixed #157` will not be accepted. -2. If the PR is not related to a github issue: `+{unique_string}.{type}.md` e.g. `+foobar.fixed.md`. - -These files can either be created manually, or using `towncrier` e.g. - - towncrier create -c 'write your description here' 157.fixed.md - -`towncrier create` also takes care of duplicates automatically (if there is more than 1 news fragment of one type -for a given github issue). +Add a [Changie](https://changie.dev) fragment to `.changes/unreleased/` for each +user-visible, documentation or infrastructure change. Go is required to run the +pinned tool, following the same setup as Backblaze/blazer: + +```sh +go run github.com/miniscruff/changie@v1.24.0 new +uv run --group nox nox -s changelog_check +``` + +Choose Added, Changed, Deprecated, Removed, Fixed, Doc, Infrastructure or Security. +Write a complete sentence that explains the resulting behavior. Include a public +issue link in the body when applicable; keep private findings out of fragments. +Use `--kind Fixed --body 'Correct the documented behavior.'` for noninteractive +creation. CI checks the fragment fields and reproduces the merged changelog. +A `-changelog` label can waive adding a fragment; validation still runs. Lockfile-only +updates and release PRs that batch fragments have their own checks. + +Release managers use `changie batch` and `changie merge` through the Nox release +session. See [RELEASE.md](RELEASE.md) for the complete process. The `.changes/v2.13.5.md` +archive preserves all earlier release notes; do not rewrite that history. ## Developer Info @@ -78,7 +72,7 @@ For example: $ nox -s format nox > Running session format - nox > Re-using existing virtual environment at .nox/format. + nox > Reusing existing virtual environment at .nox/format. ... $ nox --no-venv -s format diff --git a/README.release.md b/README.release.md deleted file mode 100644 index 9ebbd8303..000000000 --- a/README.release.md +++ /dev/null @@ -1,3 +0,0 @@ -# Release Process - -- Run `nox -s make_release_commit -- X.Y.Z` where `X.Y.Z` is the version you're releasing, and follow the instructions diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..48c04f529 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,254 @@ +# Releasing b2sdk + +A release is a reviewed preparation PR merged into `master`, followed by a +maintainer's tag on that PR's merge commit. Hatch-vcs derives the package version +from Git; do not add a static version to `pyproject.toml` or `b2sdk/version.py`. +Go and Changie manage the release notes, following Backblaze/blazer's setup. +The release tooling directly depends on PyYAML for fragment validation, replacing +Towncrier in the `release` dependency group. This does not add a runtime dependency. + +## One-time setup + +- A repository administrator must configure an **active tag ruleset** for `v*` + before enabling either protected release environment. Restrict tag creation and + updates to the release-maintainer team through an explicit bypass list; do not + grant a blanket write-role bypass. Protect published tags against update/deletion + and keep `master` behind required review and CI. Review the ruleset whenever the + release-maintainer membership changes. These are repository settings, not rules + a tag-controlled workflow can securely enforce about itself. +- Install Git, GitHub CLI, Go, uv and Python 3.14. Signed GPG tags also need GnuPG; + SSH-signed tags need Git 2.34+ and OpenSSH. Authenticate `gh` with this repository. +- Configure a PyPI **Trusted Publisher** for project `b2sdk`, owner `Backblaze`, + repository `b2-sdk-python`, workflow filename `cd.yml`, environment `pypi`. + A PyPI project owner must do this at + . Protect the GitHub + `pypi` environment with maintainer approval and tag deployment rules. OIDC replaces + the old publishing credential; remove that credential after the migration is verified. +- The publish job has `id-token: write` and explicitly enables PEP 740 attestations. + It downloads built distributions and does not build code or install build dependencies. +- Create a `release-security` GitHub environment with deployment rules allowing + only `master` and `v*` tags, required security-maintainer approval, and self-review + disabled. Configure these protections **before adding any credential**. + Ensure its gate can read Code scanning alerts, Dependabot alerts and Secret + scanning alerts. If `GITHUB_TOKEN` cannot read all three, store + `RELEASE_SECURITY_TOKEN` only as a secret of this protected environment (prefer + an installation token managed by the organization). Never store a copy as a + repository/organization secret; remove such a copy if it exists. An unreadable or malformed source **holds** + the release; disabled scanners are not silently treated as clean. +- Select and review the intended maintainer accounts in `.github/signing-maintainers`: + `goanpeca`, `ffumero2003`, `sophiecarreras`. They are GitHub usernames, not fixture keys. + +### Environment approval is an independent trust check + +Before approving **either** `release-security` or `pypi`, inspect the exact tag +commit and its release PR from a trusted checkout. Confirm its master ancestry and +successful CI independently of the candidate workflow: + +```sh +TAG=vX.Y.Z +git fetch origin master +git fetch origin tag "$TAG" +sha=$(git rev-parse "$TAG^{commit}") +git merge-base --is-ancestor "$sha" origin/master +git log -1 --oneline "$sha" +gh run list --repo Backblaze/b2-sdk-python --workflow ci.yml --branch master --commit "$sha" --limit 1 +``` + +Do not approve solely because the deployment displays a `v*` tag name. A tag can +point to unmerged code, including a modified workflow. The external tag ruleset +and the approver's independent ancestry/CI check protect the credential and OIDC +boundaries; the in-workflow preflight provides an additional normal-path check. +For a master dry run, confirm the selected master commit and CI instead. + +### Enroll signing keys before enforcement + +There is no enrolled-key baseline yet. Each maintainer must create a personal GPG +or SSH signing key, register its public key with their GitHub account and configure +Git to sign tags. Review fingerprints with the maintainer through an independent +channel. The workflow resolves GPG keys from `https://github.com/.gpg` +and SSH signing keys from `https://api.github.com/users//ssh_signing_keys` +at check time into a temporary keyring/allowed-signers file, and runs +`git verify-tag`. Do not commit a private key or test key to the maintainer list. +Register SSH keys as **Signing Keys** on GitHub. The `.keys` endpoint lists +authentication keys and is not used to authorize release signatures. + +`ENFORCE_SIGNED_TAGS` is a repository variable. Unset or `false` means **warn-only**: +an unsigned, unknown-key or unverifiable tag is reported but does not hold the +release. A restored tag pointing to a different commit than the checkout always +blocks, regardless of this variable. Once every intended release signer is enrolled +and a signed-tag check has been verified, a repository administrator enables enforcement: + +```sh +gh variable set ENFORCE_SIGNED_TAGS --repo Backblaze/b2-sdk-python --body true +``` + +With enforcement enabled, signature failures and public-key lookup failures hold +the release. GitHub account key enrollment is the trust source; there are no +committed fingerprints. Key rotation and account security therefore require review. + +## Security gate and manual review + +Run the same gate used by delivery before planning a release: + +```sh +uv run --group nox python scripts/release_security.py Backblaze/b2-sdk-python +``` + +It holds for medium/high/critical CodeQL security alerts, medium/high/critical +Dependabot alerts with an available fix, and every open secret-scanning alert +(that API has no severity field). Code-scanning alerts from other tools are outside +this automated CodeQL policy. Secret-scanning requests set `hide_secret=true`. +Low alerts and dependencies without a patched +version do not hold this gate. The output contains source counts only. Resolve or +have an authorized owner dismiss findings in their source system with a reason; +the gate has no finding waiver flag. An API error is `UNVERIFIED` and exits nonzero. + +The public workflow cannot query the organization's private Snyk tracking mirror. +Before approving the release, a maintainer must manually review unresolved +`severity:`-labelled findings in `Backblaze/b2-sdk-python_private`, following the +Confluence page **Security considerations for Backblaze Labs projects**. Coordinate +any medium/high/critical findings with the security owner. Keep private finding +details out of public issues, PRs, release notes and workflow logs. + +## Plan and prepare the release PR + +1. Assign intended public work to a release milestone, review the merged changes + since the last tag, and agree the semantic version. Breaking changes require a + major, compatible features a minor, fixes/tooling a patch; choose deliberately + when the release has mixed kinds. Changie's `auto` is a preview, not approval. +2. Start from clean, current `master`; this is a hard preparation check: + + ```sh + git fetch origin --tags + git switch master + git pull --ff-only origin master + git status --short + gh run list --repo Backblaze/b2-sdk-python --branch master --workflow ci.yml --limit 1 + git log --oneline vLAST..origin/master + uv run --group nox python scripts/release_security.py Backblaze/b2-sdk-python + ``` + + Require successful CI for the actual current master commit. A pending, absent + or failed run is a hold; do not release while main is red. +3. Review `.changes/unreleased/` and prepare an explicit version: + + ```sh + go run github.com/miniscruff/changie@v1.24.0 batch patch --dry-run + uv run --locked --group nox nox -vs make_release_commit -- X.Y.Z + git switch -c release/vX.Y.Z + ``` + + The Nox command requires a clean master worktree, including untracked files; + rejects an existing tag; runs `changie batch vX.Y.Z` then `changie merge`; and + stops. It never commits, tags or publishes. It creates `.changes/vX.Y.Z.md`, + consumes the unreleased fragments and updates `CHANGELOG.md`. Historical notes + remain in `.changes/v2.13.5.md`; do not rewrite that archive. Review the new + `## [X.Y.Z]` section and regenerate with `changie merge` after editing its batch. + Supported prerelease suffixes are `-aN`, `-bN` and `-rcN`, also accepting a dot + before `N` (`-a.N`, `-b.N`, `-rc.N`). This preserves the repository's historical + `v1.0.0-rc1` tag style. Numeric release components accept ASCII digits, including + leading zeroes; `N` is nonnegative. This is a deliberately restricted, + Python-compatible interface, not arbitrary SemVer identifiers or build metadata. + All three release entry points use the same validator. For example, run + `uv run --locked --group nox nox -vs make_release_commit -- 2.14.0-rc.1`. + Keep that exact suffix in the branch, Changie batch and tag (`v2.14.0-rc.1`); + preparing `2.14.0` does not prepare the notes for `2.14.0-rc.1`. +4. Run local verification with no production credentials in the environment: + + ```sh + uv run --locked --group nox nox -vs lint unit build doc changelog_check + ``` + + `unit` covers v0–v3 on the configured Python interpreters. CI additionally + exercises the OS/Python matrix, CLI compatibility and credentialed integration + tests when secrets are available. A local skip is not evidence for a live test. +5. Run the local customer health check against a fresh loopback simulator. From + the central harness checkout (see `.sdkharness/README.md` for manual simulator + startup and the fixed test credentials): + + ```sh + B2SIM_SOURCE=pinned sdkharness/bin/run-health-checks.sh --target simulator --versions head b2-sdk-python + ``` + + This requires the harness to have a clean SDK clone at the release candidate. + Verify the printed revision; a PASS for another checkout does not verify this + candidate. The repository-owned health executable is `.sdkharness/tests/health-golden-path`. +6. Review the diff, commit and open the release PR: + + ```sh + git add CHANGELOG.md .changes + git commit -m 'chore(release): X.Y.Z' + git push -u origin release/vX.Y.Z + gh pr create --base master --title 'chore(release): X.Y.Z' --assignee @me + ``` + + Include the changelog, exact commit scope and verification evidence; request a + human reviewer. If the scope is tooling-only, say so. Wait for approval and + green PR CI. The maintainer merges; release preparation does not authorize a tag. + +## Non-publishing workflow dry run + +After the dispatch-enabled `cd.yml` is on the default branch: + +```sh +gh workflow run cd.yml --repo Backblaze/b2-sdk-python --ref master -f dry_run=true +``` + +The dry run checks current master CI and the live security sources, assembles a +disposable Changie batch, extracts its notes, builds an untagged development +distribution, smoke-tests the installed v0–v3 modules and uploads CI artifacts. +Neither the PyPI publisher nor `gh release create/edit/upload` runs on dispatch; +`dry_run=false` is rejected. No tag is created. This does not prove PyPI's OIDC +configuration or attestation upload; those are only exercised by an actual approved +publish. TestPyPI is not configured by this workflow and dispatch does not publish there. +GitHub may reject dispatch before the workflow trigger exists on master; record +that as a verification limit rather than bypassing the gate. The protected +security environment intentionally rejects dispatches on release/feature branches; +tag deployments additionally require the tag ruleset and independent environment +approval described above. Environment ref patterns alone do not establish that a +tag's commit was merged. + +## Tag and publish (maintainer only) + +After the preparation PR merges, fetch its exact merge commit and verify its CI. +Tag that commit, which must be reachable from current `origin/master`: + +```sh +git fetch origin --tags +git switch master +git pull --ff-only origin master +sha=$(gh pr view RELEASE_PR --repo Backblaze/b2-sdk-python --json mergeCommit --jq .mergeCommit.oid) +git log -1 --oneline "$sha" +git merge-base --is-ancestor "$sha" origin/master +git tag -s vX.Y.Z "$sha" -m 'vX.Y.Z' +uv run --group nox python scripts/verify_release_tag.py vX.Y.Z +git push origin refs/tags/vX.Y.Z +``` + +Configure `gpg.format=ssh` and `user.signingkey` for SSH, or the GPG signing key for +GPG, before `git tag -s`. The tag workflow requires successful master and tagged +commit CI, master ancestry, the configured signature gate, a clean security gate, +matching Changie version/notes and no remaining fragments before publishing. +Stable and supported prerelease tags publish to PyPI through OIDC with PEP 740 +attestations, preserving uploads for the established digit-terminated prerelease +tag style. Prereleases are now correctly marked as GitHub prereleases instead of +using the previous last-character classification. The publishing job requires the +protected `pypi` environment for both stable and prerelease uploads. + +After successful PyPI upload, the workflow creates the GitHub release with the +exact changelog section. Re-running the GitHub-release job edits existing notes +and replaces assets via `gh release upload --clobber`. There are no long-lived +PyPI credentials in the workflow. Confirm the release, package version, provenance +and a fresh install before declaring the release complete. + +## Recovery + +- Before a tag is pushed: fix the PR, repeat checks and obtain approval. +- A security, CI, ancestry, signature or unreadable-source hold requires resolving + the underlying condition and re-running checks; never disable a gate to ship. +- After a tag is public: do not move or delete it. Use a new version for a new build. +- If publication fails, inspect registry state and attestations before retrying. + PyPI duplicates fail rather than silently accepting a different artifact. If + PyPI already has the correct files but GitHub release creation failed, complete + the GitHub release with the verified artifacts and extracted notes; do not + rebuild and overwrite a published package. Record the recovery in the release PR. diff --git a/b2sdk/_internal/scan/policies.py b/b2sdk/_internal/scan/policies.py index 41575346a..d42d2d7e0 100644 --- a/b2sdk/_internal/scan/policies.py +++ b/b2sdk/_internal/scan/policies.py @@ -28,7 +28,7 @@ class RegexSet: def __init__(self, regex_iterable): """ - :param regex_iterable: an interable which yields regexes + :param regex_iterable: an iterable which yields regexes """ self._compiled_list = [re.compile(r) for r in regex_iterable] diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 03c33fe9a..34f8eb175 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -49,7 +49,7 @@ For example:: $ nox -s format nox > Running session format - nox > Re-using existing virtual environment at .nox/format. + nox > Reusing existing virtual environment at .nox/format. ... $ nox --no-venv -s format diff --git a/noxfile.py b/noxfile.py index 418ef765f..4978ed20c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -12,13 +12,9 @@ import os import pathlib import platform -import re -import subprocess import nox -UPSTREAM_REPO_URL = 'git@github.com:Backblaze/b2-sdk-python.git' - CI = os.environ.get('CI') is not None NOX_PYTHONS = os.environ.get('NOX_PYTHONS') _NOX_EXTRAS = os.environ.get('NOX_EXTRAS') @@ -56,7 +52,7 @@ def _detect_python_nox_id() -> str: PYTHON_VERSIONS[-1], ) -PY_PATHS = ['b2sdk', 'test', 'noxfile.py'] +PY_PATHS = ['b2sdk', 'test', 'scripts', 'noxfile.py'] nox.options.default_venv_backend = 'uv' nox.options.reuse_existing_virtualenvs = True @@ -210,7 +206,7 @@ def build(session): "'v2': pathlib.Path(v2.__file__).resolve(), " "'v3': pathlib.Path(v3.__file__).resolve()}; " 'print(module_files); ' - "assert all(not path.is_relative_to(source_root) for path in module_files.values()), " + 'assert all(not path.is_relative_to(source_root) for path in module_files.values()), ' "f'Imported modules from checkout: {module_files!r}'; " "assert all('site-packages' in path.parts for path in module_files.values()), " "f'Imported modules from an unexpected location: {module_files!r}'" @@ -261,158 +257,14 @@ def doc_cover(session): @nox.session(python=PYTHON_DEFAULT_VERSION) def make_release_commit(session): - """ - Runs `towncrier build`, commits changes, tags, all that is left to do is pushing - """ - if session.posargs: - version = session.posargs[0] - else: - session.error('Provide -- {release_version} (X.Y.Z - without leading "v")') - - if not re.match(r'^\d+\.\d+\.\d+$', version): - session.error( - f'Provided version="{version}". Version must be of the form X.Y.Z where ' - f'X, Y and Z are integers' - ) - - local_changes = subprocess.check_output(['git', 'diff', '--stat']) - if local_changes: - session.error('Uncommitted changes detected') - - current_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode() - if current_branch != 'master': - session.log('WARNING: releasing from a branch different than master') - - uv_install(session, groups=('release',)) - session.run('towncrier', 'build', '--yes', '--version', version) - - session.log( - f'CHANGELOG updated, changes ready to commit and push\n' - f' git remote add upstream {UPSTREAM_REPO_URL!r} 2>/dev/null || git remote get-url upstream\n' - f' git commit -m "release {version}"\n' - f' git push upstream {current_branch}\n' - f'Wait for a CI workflow to complete successfully, before triggering CD by pushing a tag.\n' - f' git tag v{version}\n' - f' git push upstream v{version}\n' - f'Wait for a CD workflow to complete successfully, indicates the release is done.' - ) - - -def load_allowed_change_types( - project_toml: pathlib.Path = pathlib.Path('./pyproject.toml'), -) -> set[str]: - """ - Load the list of allowed change types from the pyproject.toml file. - """ - import tomllib - - configuration = tomllib.loads(project_toml.read_text()) - return set(entry['directory'] for entry in configuration['tool']['towncrier']['type']) - - -def is_changelog_filename_valid(filename: str, allowed_change_types: set[str]) -> tuple[bool, str]: - """ - Validates whether the given filename matches our rules. - Provides information about why it doesn't match them. - """ - error_reasons = [] - - wanted_extension = 'md' - try: - description, change_type, extension = filename.rsplit('.', maxsplit=2) - except ValueError: - # Not enough values to unpack. - return False, 'Doesn\'t follow the "..md" pattern.' - - # Check whether the filename ends with .md. - if extension != wanted_extension: - error_reasons.append(f"Doesn't end with {wanted_extension} extension.") - - # Check whether the change type is valid. - if change_type not in allowed_change_types: - error_reasons.append( - f"Change type '{change_type}' doesn't match allowed types: {allowed_change_types}." - ) - - # Check whether the description makes sense. - try: - int(description) - except ValueError: - if description[0] != '+': - error_reasons.append("Doesn't start with a number nor a plus sign.") - - return len(error_reasons) == 0, ' / '.join(error_reasons) if error_reasons else '' - - -def is_changelog_entry_valid(file_content: str) -> tuple[bool, str]: - """ - We expect the changelog entry to be a valid sentence in the English language. - This includes, but not limits to, providing a capital letter at the start - and the full-stop character at the end. - - Note: to do this "properly", tools like `nltk` and `spacy` should be used. - """ - error_reasons = [] - - # Check whether the first character is a capital letter. - # Not allowing special characters nor numbers at the very start. - if not file_content[0].isalpha() or not file_content[0].isupper(): - error_reasons.append('The first character is not a capital letter.') - - # Check if the last character is a full-stop character. - if file_content.strip()[-1] != '.': - error_reasons.append('The last character is not a full-stop character.') - - return len(error_reasons) == 0, ' / '.join(error_reasons) if error_reasons else '' + """Prepare a Changie batch on clean master; never commit, tag or publish.""" + if len(session.posargs) != 1: + session.error('Provide -- X.Y.Z (without leading v)') + session.run('python', 'scripts/prepare_release.py', session.posargs[0]) @nox.session(python=PYTHON_DEFAULT_VERSION) -def towncrier_check(session): - """ - Check whether all the entries in the changelog.d follow the expected naming convention - as well as some basic rules as to their format. - """ - expected_non_md_files = {'.gitkeep'} - allowed_change_types = load_allowed_change_types() - - is_error = False - - for filename in pathlib.Path('./changelog.d/').glob('*'): - # If that's an expected file, it's all right. - if filename.name in expected_non_md_files: - continue - - # Check whether the file matches the expected pattern. - is_valid, error_message = is_changelog_filename_valid(filename.name, allowed_change_types) - if not is_valid: - session.log(f"File {filename.name} doesn't match the expected pattern: {error_message}") - is_error = True - continue - - # Check whether the file isn't too big. - if filename.lstat().st_size > 16 * 1024: - session.log( - f'File {filename.name} content is too big – it should be smaller than 16kB.' - ) - is_error = True - continue - - # Check whether the file can be loaded as UTF-8 file. - try: - file_content = filename.read_text(encoding='utf-8') - except UnicodeDecodeError: - session.log(f'File {filename.name} is not a valid UTF-8 file.') - is_error = True - continue - - # Check whether the content of the file is anyhow valid. - is_valid, error_message = is_changelog_entry_valid(file_content) - if not is_valid: - session.log(f'File {filename.name} is not a valid changelog entry: {error_message}') - is_error = True - continue - - if is_error: - session.error( - 'Found errors in the changelog.d directory. Check logs above for more information' - ) +def changelog_check(session): + """Validate Changie fragments, generated history and optional PR coverage.""" + uv_install(session, groups=('release',)) + session.run('python', 'scripts/check_changelog.py', *session.posargs) diff --git a/pyproject.toml b/pyproject.toml index 9e80c3e3e..cd706ca68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ lint = [ "setuptools<81", # liccheck 0.9.2 imports pkg_resources, removed in setuptools 81 ] test = [ + {include-group = "release"}, "pip>=26.2.2", "coverage>=7.13.5,<8", "pytest>=9.0.3,<10", @@ -70,7 +71,7 @@ test = [ "responses>=0.26.0,<1", ] release = [ - "towncrier==23.11.0", + "pyyaml>=6.0.3,<7", ] doc = [ "sadisplay>=0.4.9", @@ -161,46 +162,3 @@ quote-style = "single" "b2sdk/v*/**" = ["I", "F403", "F405"] "b2sdk/_v*/**" = ["I", "F403", "F405"] "test/**" = ["D", "F403", "F405"] - -[tool.towncrier] -directory = "changelog.d" -filename = "CHANGELOG.md" -start_string = "\n" -underlines = ["", "", ""] -title_format = "## [{version}](https://github.com/Backblaze/b2-sdk-python/releases/tag/v{version}) - {project_date}" -issue_format = "[#{issue}](https://github.com/Backblaze/b2-sdk-python/issues/{issue})" - -[[tool.towncrier.type]] -directory = "removed" -name = "Removed" -showcontent = true - -[[tool.towncrier.type]] -directory = "changed" -name = "Changed" -showcontent = true - -[[tool.towncrier.type]] -directory = "fixed" -name = "Fixed" -showcontent = true - -[[tool.towncrier.type]] -directory = "deprecated" -name = "Deprecated" -showcontent = true - -[[tool.towncrier.type]] -directory = "added" -name = "Added" -showcontent = true - -[[tool.towncrier.type]] -directory = "doc" -name = "Doc" -showcontent = true - -[[tool.towncrier.type]] -directory = "infrastructure" -name = "Infrastructure" -showcontent = true diff --git a/scripts/check_changelog.py b/scripts/check_changelog.py new file mode 100644 index 000000000..86feca81f --- /dev/null +++ b/scripts/check_changelog.py @@ -0,0 +1,85 @@ +###################################################################### +# +# File: scripts/check_changelog.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Validate fragments and require a reproducible Changie merge without modifying files.""" + +from __future__ import annotations + +import argparse +import datetime +import subprocess +from pathlib import Path + +CHANGIE = ['go', 'run', 'github.com/miniscruff/changie@v1.24.0'] +KINDS = {'Added', 'Changed', 'Deprecated', 'Removed', 'Fixed', 'Doc', 'Infrastructure', 'Security'} + + +def validate_fragment(value): + if not isinstance(value, dict) or value.get('kind') not in KINDS: + raise ValueError('A fragment needs a configured kind') + body = value.get('body') + if not isinstance(body, str) or not body.strip(): + raise ValueError('A fragment needs a nonempty body') + time = value.get('time') + if not isinstance(time, (datetime.datetime, str)): + raise ValueError('A fragment needs a timestamp') + datetime.datetime.fromisoformat(str(time).replace('Z', '+00:00')) + + +def main(): + import yaml + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base') + parser.add_argument('--allow-missing-fragment', action='store_true') + args = parser.parse_args() + root = Path.cwd() + has_fragments = False + for path in (root / '.changes/unreleased').iterdir(): + if path.name == '.gitkeep': + continue + if path.is_symlink() or not path.is_file() or path.suffix not in {'.yaml', '.yml'}: + raise ValueError('Unreleased entries must be regular YAML files') + if path.stat().st_size > 16 * 1024: + raise ValueError('Fragment exceeds 16 KiB') + validate_fragment(yaml.safe_load(path.read_text())) + has_fragments = True + merged = subprocess.run( + [*CHANGIE, 'merge', '--dry-run'], check=True, capture_output=True, text=True + ) + if merged.stdout != (root / 'CHANGELOG.md').read_text(): + raise ValueError('CHANGELOG.md differs from changie merge; regenerate it') + if has_fragments: + subprocess.run([*CHANGIE, 'batch', 'patch', '--dry-run'], check=True, capture_output=True) + if args.base and not args.allow_missing_fragment: + diff = subprocess.run( + ['git', 'diff', '--name-status', f'{args.base}...HEAD'], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + added = [line.split('\t')[-1] for line in diff if line.startswith('A\t')] + paths = {line.split('\t')[-1] for line in diff} + fragment = any( + path.startswith('.changes/unreleased/') and path.endswith(('.yaml', '.yml')) + for path in added + ) + release = 'CHANGELOG.md' in paths and any( + Path(path).parent == Path('.changes') + and path.startswith('.changes/v') + and path.endswith('.md') + for path in added + ) + if not fragment and not release and paths != {'uv.lock'}: + raise ValueError('Add a Changie fragment, or batch a release with its changelog') + print('Changelog checks passed') + + +if __name__ == '__main__': + main() diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py new file mode 100644 index 000000000..29a9943a5 --- /dev/null +++ b/scripts/prepare_release.py @@ -0,0 +1,56 @@ +###################################################################### +# +# File: scripts/prepare_release.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Prepare release notes on a clean master; never commit, tag, or publish.""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +CHANGIE = ['go', 'run', 'github.com/miniscruff/changie@v1.24.0'] +VERSION_PATTERN = r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:a|b|rc)\.?(?:0|[1-9][0-9]*))?' + + +def prepare(version, root, run=subprocess.run): + if not re.fullmatch(VERSION_PATTERN, version): + raise ValueError( + 'Version must be X.Y.Z, optionally followed by -a.N, -b.N or -rc.N ' + '(the dot before N is optional); omit the leading v' + ) + + def git(*args): + return run( + ['git', *args], cwd=root, check=True, capture_output=True, text=True + ).stdout.strip() + + if git('branch', '--show-current') != 'master': + raise RuntimeError('Release preparation requires master') + if git('status', '--porcelain'): + raise RuntimeError( + 'Release preparation requires a clean worktree, including untracked files' + ) + if git('tag', '--list', f'v{version}'): + raise RuntimeError('That release tag already exists') + run([*CHANGIE, 'batch', f'v{version}'], cwd=root, check=True) + run([*CHANGIE, 'merge'], cwd=root, check=True) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + sys.exit('Usage: python scripts/prepare_release.py X.Y.Z') + try: + prepare(sys.argv[1], Path.cwd()) + except (ValueError, RuntimeError, subprocess.CalledProcessError) as error: + sys.exit(str(error)) + print( + 'Release notes prepared. Create a release branch and PR; follow RELEASE.md after approval.' + ) diff --git a/scripts/release-notes.awk b/scripts/release-notes.awk new file mode 100644 index 000000000..1aecbb0d6 --- /dev/null +++ b/scripts/release-notes.awk @@ -0,0 +1,10 @@ +# Extract one exact version. Treat changelog text as data. +/^## \[/ { + if (active) exit + heading = $0 + sub(/^## \[/, "", heading) + sub(/\].*$/, "", heading) + if (heading == version) { active = 1; found = 1; next } +} +active { print } +END { if (!found) exit 1 } diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py new file mode 100644 index 000000000..de0115c74 --- /dev/null +++ b/scripts/release_preflight.py @@ -0,0 +1,96 @@ +###################################################################### +# +# File: scripts/release_preflight.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Require successful master CI and, for a real release, a tag on master history.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys + +from prepare_release import VERSION_PATTERN + + +class ReleaseHold(RuntimeError): + """An actionable reason constructed by this gate, never remote error content.""" + + +def failure_reason(error): + if isinstance(error, ReleaseHold): + reason = str(error) + elif isinstance(error, subprocess.CalledProcessError): + if error.cmd[:2] == ['git', 'merge-base']: + reason = 'Release commit is not on master history' + elif error.cmd[0] == 'gh': + reason = 'Cannot read required CI metadata' + else: + reason = 'Cannot fetch or resolve the release and master revisions' + else: + reason = 'Required release or CI metadata is missing or malformed' + return f'Release preflight failed: {reason}' + + +def require_green_ci(repo, sha): + result = subprocess.run( + ['gh', 'api', f'repos/{repo}/actions/workflows/ci.yml/runs?head_sha={sha}&per_page=100'], + check=True, + capture_output=True, + text=True, + ) + runs = [ + run + for run in json.loads(result.stdout)['workflow_runs'] + if run['head_sha'] == sha and run['head_branch'] == 'master' and run['event'] == 'push' + ] + if not runs or runs[0]['status'] != 'completed' or runs[0]['conclusion'] != 'success': + raise ReleaseHold( + 'The latest master CI run for this commit must have completed successfully' + ) + + +def main(): + repo = os.environ['GITHUB_REPOSITORY'] + if repo != 'Backblaze/b2-sdk-python': + raise ReleaseHold('Release workflow is restricted to Backblaze/b2-sdk-python') + subprocess.run(['git', 'fetch', '--no-tags', 'origin', 'master'], check=True) + + def git(*args): + return subprocess.run( + ['git', *args], check=True, capture_output=True, text=True + ).stdout.strip() + + master = git('rev-parse', 'refs/remotes/origin/master') + require_green_ci(repo, master) + if os.environ['GITHUB_EVENT_NAME'] == 'workflow_dispatch': + if os.environ.get('INPUT_DRY_RUN') != 'true': + raise ReleaseHold('Manual dispatch supports dry_run=true only; it never publishes') + print('Non-publishing dry run; master CI is green.') + return + if os.environ.get('GITHUB_REF_TYPE') != 'tag': + raise ReleaseHold('A release requires a tag push') + tag = os.environ['GITHUB_REF_NAME'] + if not re.fullmatch('v' + VERSION_PATTERN, tag): + raise ReleaseHold('Invalid release tag') + head = git('rev-parse', 'HEAD') + if git('rev-parse', f'refs/tags/{tag}^{{commit}}') != head: + raise ReleaseHold('Checked-out revision differs from the release tag') + subprocess.run(['git', 'merge-base', '--is-ancestor', head, master], check=True) + require_green_ci(repo, head) + print('Release tag is on master history and both release and master CI are green.') + + +if __name__ == '__main__': + try: + main() + except (KeyError, ValueError, RuntimeError, subprocess.SubprocessError) as error: + # Do not echo API payloads or credential-bearing stderr. + sys.exit(failure_reason(error)) diff --git a/scripts/release_security.py b/scripts/release_security.py new file mode 100644 index 000000000..cc9fd3d42 --- /dev/null +++ b/scripts/release_security.py @@ -0,0 +1,80 @@ +###################################################################### +# +# File: scripts/release_security.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Fail closed on the public GitHub security sources; print counts, never alert content.""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys + +BLOCKING = {'medium', 'high', 'critical'} + + +def github_api(endpoint): + result = subprocess.run( + ['gh', 'api', '--paginate', '--slurp', endpoint], + check=True, + capture_output=True, + text=True, + ) + pages = json.loads(result.stdout) + if not isinstance(pages, list) or not all(isinstance(page, list) for page in pages): + raise ValueError('Invalid paginated response') + return [item for page in pages for item in page] + + +def check(repo, api=github_api): + if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo): + raise ValueError('Expected owner/repository') + holds = [] + for source in ('dependabot', 'code-scanning', 'secret-scanning'): + try: + endpoint = f'repos/{repo}/{source}/alerts?state=open&per_page=100' + if source == 'secret-scanning': + endpoint += '&hide_secret=true' + items = api(endpoint) + if not isinstance(items, list) or not all(isinstance(item, dict) for item in items): + raise ValueError('Invalid alert response') + count = 0 + for item in items: + if source == 'dependabot': + severity = item['security_advisory']['severity'] + if severity not in BLOCKING | {'low'}: + raise ValueError('Unknown security severity') + count += severity in BLOCKING and bool( + item['security_vulnerability']['first_patched_version'] + ) + elif source == 'code-scanning': + if item['tool']['name'] != 'CodeQL': + continue + rule = item['rule'] + severity = rule.get('security_severity_level') + if severity not in BLOCKING | {'low', None}: + raise ValueError('Unknown security severity') + if severity is None and 'security' in rule.get('tags', []): + raise ValueError('Security alert has no severity') + count += severity in BLOCKING + else: + # The secret-scanning API has no severity. Every open credential exposure holds. + count += 1 + print(f'{source}: {len(items)} open; {count} blocking') + if count: + holds.append(f'{source}: {count} blocking alert(s)') + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, RuntimeError): + holds.append(f'{source}: UNVERIFIED; cannot read or classify the required source') + return holds + + +if __name__ == '__main__': + for hold in (holds := check(sys.argv[1] if len(sys.argv) == 2 else 'Backblaze/b2-sdk-python')): + print(f'HOLD: {hold}') + sys.exit(bool(holds)) diff --git a/scripts/verify_release_tag.py b/scripts/verify_release_tag.py new file mode 100644 index 000000000..e82985c6a --- /dev/null +++ b/scripts/verify_release_tag.py @@ -0,0 +1,164 @@ +###################################################################### +# +# File: scripts/verify_release_tag.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Verify tags using the enrolled maintainers' GitHub public GPG and SSH keys.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import urllib.error +import urllib.request +from pathlib import Path + +from prepare_release import VERSION_PATTERN + + +def enforcing(value): + normalized = value.strip().lower() + if normalized not in {'', 'false', 'off', '0', 'true', 'on', '1'}: + raise ValueError('ENFORCE_SIGNED_TAGS must be true or false') + return normalized in {'true', 'on', '1'} + + +def fetch_keys(url): + try: + with urllib.request.urlopen(url, timeout=15) as response: + content = response.read(1024 * 1024 + 1) + if len(content) > 1024 * 1024: + raise ValueError('Public key response exceeds size limit') + return content.decode('utf-8') + except urllib.error.HTTPError as error: + if error.code == 404: + return '' + raise + + +def verify(tag, root, *, enforce, maintainers, fetch=fetch_keys): + verified = False + try: + if not re.fullmatch('v' + VERSION_PATTERN, tag): + raise ValueError('Invalid release tag') + commits = [ + subprocess.run( + ['git', 'rev-parse', ref], cwd=root, check=True, capture_output=True, text=True + ).stdout.strip() + for ref in (f'refs/tags/{tag}^{{commit}}', 'HEAD') + ] + if commits[0] != commits[1] or ( + os.getenv('GITHUB_SHA') and commits[0] != os.environ['GITHUB_SHA'] + ): + print('::error::Restored tag does not point to the checked-out release commit.') + return False + if not maintainers or any(not re.fullmatch(r'[A-Za-z0-9-]+', name) for name in maintainers): + raise ValueError('Invalid maintainer list') + with tempfile.TemporaryDirectory(prefix='b2sdk-signers-') as temporary: + home = Path(temporary) + home.chmod(0o700) + ssh_keys = [] + for name in maintainers: + gpg_keys = fetch(f'https://github.com/{name}.gpg') + if gpg_keys and shutil.which('gpg'): + # GitHub returns an armored stub for accounts with no GPG keys. + # An unsuccessful import must still allow an enrolled SSH key. + subprocess.run( + ['gpg', '--batch', '--homedir', str(home), '--import'], + input=gpg_keys, + text=True, + capture_output=True, + check=False, + ) + page = 1 + while True: + keys = json.loads( + fetch( + f'https://api.github.com/users/{name}/ssh_signing_keys?per_page=100&page={page}' + ) + ) + if not isinstance(keys, list): + raise ValueError('Invalid SSH signing-key response') + for item in keys: + key = item.get('key') if isinstance(item, dict) else None + if ( + not isinstance(key, str) + or not key.strip() + or len(key.splitlines()) != 1 + ): + raise ValueError('Invalid SSH signing key') + ssh_keys.append(f'{name} {key.strip()}') + if len(keys) < 100: + break + page += 1 + signers = home / 'allowed_signers' + signers.write_text('\n'.join(ssh_keys) + '\n') + commands = [ + [ + 'git', + '-c', + 'gpg.format=openpgp', + '-c', + f'gpg.ssh.allowedSignersFile={signers}', + 'verify-tag', + tag, + ], + [ + 'git', + '-c', + 'gpg.format=ssh', + '-c', + f'gpg.ssh.allowedSignersFile={signers}', + 'verify-tag', + tag, + ], + ] + for command in commands: + result = subprocess.run( + command, + cwd=root, + env={**os.environ, 'GNUPGHOME': str(home)}, + capture_output=True, + text=True, + ) + if result.returncode == 0: + verified = True + break + except (OSError, ValueError, subprocess.SubprocessError, urllib.error.URLError): + verified = False + if verified: + print('Tag signature verified against an enrolled GitHub maintainer key.') + else: + level = 'error' if enforce else 'warning' + print( + f'::{level}::Tag signature unverified; enroll maintainer keys before enabling enforcement.' + ) + return verified or not enforce + + +if __name__ == '__main__': + root = Path(__file__).resolve().parents[1] + names = [ + line.strip() + for line in (root / '.github/signing-maintainers').read_text().splitlines() + if line.strip() and not line.startswith('#') + ] + try: + result = verify( + sys.argv[1], + root, + enforce=enforcing(os.getenv('ENFORCE_SIGNED_TAGS', '')), + maintainers=names, + ) + except (IndexError, ValueError): + sys.exit('Invalid tag or ENFORCE_SIGNED_TAGS setting') + sys.exit(not result) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index 4b3ec1d11..e564f953f 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -100,7 +100,7 @@ def test_small_unverified(self): @pytest.mark.parametrize('size_multiplier', [1, 100]) def test_gzip(b2_auth_data, bucket, tmp_path, b2_api, size_multiplier, realm): - """Test downloading gzipped files of varius sizes with and without content-encoding.""" + """Test downloading gzipped files of various sizes with and without content-encoding.""" source_file = tmp_path / 'compressed_file.gz' downloaded_uncompressed_file = tmp_path / 'downloaded_uncompressed_file' downloaded_compressed_file = tmp_path / 'downloaded_compressed_file' diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index 86ff96fc1..82077a5d8 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -2863,9 +2863,9 @@ def test_download_to_non_seekable_file(self): file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') non_seekable_strategies = [ - strat - for strat in self.bucket.api.services.download_manager.strategies - if not isinstance(strat, ParallelDownloader) + strategy + for strategy in self.bucket.api.services.download_manager.strategies + if not isinstance(strategy, ParallelDownloader) ] context = ( contextlib.nullcontext() @@ -2888,9 +2888,9 @@ def test_download_to_seekable_but_no_read_file(self): file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') non_seekable_strategies = [ - strat - for strat in self.bucket.api.services.download_manager.strategies - if not isinstance(strat, ParallelDownloader) + strategy + for strategy in self.bucket.api.services.download_manager.strategies + if not isinstance(strategy, ParallelDownloader) ] context = ( contextlib.nullcontext() diff --git a/test/unit/internal/transfer/downloader/test_parallel.py b/test/unit/internal/transfer/downloader/test_parallel.py index eda3d4be6..ea7ca95b2 100644 --- a/test/unit/internal/transfer/downloader/test_parallel.py +++ b/test/unit/internal/transfer/downloader/test_parallel.py @@ -158,7 +158,7 @@ def iter_content(chunk_size=1, decode_unicode=False): def test_download_file__data_stream_error__multiple_errors_recovery( apiver_module, b2api, bucket, downloader, output_file ): - """Test downloader handles multiple half-failed requests and still downlaods entire file.""" + """Test downloader handles multiple half-failed requests and still downloads entire file.""" # This works since each part is attempted up to 15 times before giving up file_size = 100 mock_response, download_version = mock_download_response_factory( diff --git a/test/unit/test_release_tooling.py b/test/unit/test_release_tooling.py new file mode 100644 index 000000000..f28fcb54b --- /dev/null +++ b/test/unit/test_release_tooling.py @@ -0,0 +1,610 @@ +###################################################################### +# +# File: test/unit/test_release_tooling.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations + +import json +import runpy +import shlex +import shutil +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +ROOT = Path(__file__).parents[2] + + +def tooling(name): + with patch.object(sys, 'path', [str(ROOT / 'scripts'), *sys.path]): + return runpy.run_path(str(ROOT / 'scripts' / f'{name}.py')) + + +@pytest.fixture +def release_repo(tmp_path, monkeypatch): + # Temporary repositories have their own commits, independent of the CI checkout. + monkeypatch.delenv('GITHUB_SHA', raising=False) + + def git(*args): + return subprocess.run( + ['git', *args], cwd=tmp_path, check=True, capture_output=True, text=True + ) + + git('init', '--initial-branch=master') + git('config', 'user.name', 'Release test') + git('config', 'user.email', 'release-test@example.invalid') + git('config', 'commit.gpgsign', 'false') + git('config', 'tag.gpgsign', 'false') + git('commit', '--allow-empty', '-m', 'Initial fixture') + return tmp_path, git + + +def test_release_preparation_refuses_other_branches(release_repo): + root, git = release_repo + git('switch', '-c', 'feature') + with pytest.raises(RuntimeError, match='master'): + tooling('prepare_release')['prepare']('2.13.5', root) + + +def test_release_preparation_refuses_untracked_changes(release_repo): + root, _ = release_repo + (root / 'pending').write_text('pending') + with pytest.raises(RuntimeError, match='clean'): + tooling('prepare_release')['prepare']('2.13.5', root) + + +@pytest.mark.parametrize('version', ['v2.13.5', '2.13', '2.13.5;echo unsafe', '2.13.5\n']) +def test_release_preparation_rejects_invalid_versions(release_repo, version): + root, _ = release_repo + with pytest.raises(ValueError, match='X.Y.Z'): + tooling('prepare_release')['prepare'](version, root) + + +def test_release_preparation_refuses_an_existing_tag(release_repo): + root, git = release_repo + git('tag', 'v2.13.5') + with pytest.raises(RuntimeError, match='exists'): + tooling('prepare_release')['prepare']('2.13.5', root) + + +def test_release_preparation_batches_then_merges_without_tagging(release_repo): + root, git = release_repo + calls = [] + + def run(args, **kwargs): + if args[0] == 'go': + calls.append(args) + return subprocess.CompletedProcess(args, 0, '') + return subprocess.run(args, **kwargs) + + tooling('prepare_release')['prepare']('2.13.5', root, run=run) + assert [call[3:] for call in calls] == [['batch', 'v2.13.5'], ['merge']] + assert git('tag', '--list').stdout == '' + + +@pytest.mark.parametrize('version', ['2.14.0-a.1', '2.14.0-b.1', '2.14.0-rc.1', '2.14.0-rc1']) +def test_release_preparation_preserves_the_prerelease_version(release_repo, version): + root, _ = release_repo + calls = [] + + def run(args, **kwargs): + if args[0] == 'go': + calls.append(args) + return subprocess.CompletedProcess(args, 0, '') + return subprocess.run(args, **kwargs) + + tooling('prepare_release')['prepare'](version, root, run=run) + assert calls[0][3:] == ['batch', f'v{version}'] + + +def test_release_preparation_preserves_zero_padded_components(release_repo): + root, _ = release_repo + calls = [] + + def run(args, **kwargs): + if args[0] == 'go': + calls.append(args) + return subprocess.CompletedProcess(args, 0, '') + return subprocess.run(args, **kwargs) + + tooling('prepare_release')['prepare']('02.13.05', root, run=run) + assert [call[3:] for call in calls] == [['batch', 'v02.13.05'], ['merge']] + + +@pytest.mark.parametrize( + 'version', ['2.14.0-preview.1', '2.14.0-rc-1', '2.14.0-rc..1', '2.1\u0664.0'] +) +def test_release_preparation_rejects_unsupported_prerelease_schemes(release_repo, version): + root, _ = release_repo + + def run(args, **kwargs): + if args[0] == 'go': + return subprocess.CompletedProcess(args, 0, '') + return subprocess.run(args, **kwargs) + + with pytest.raises(ValueError, match='X.Y.Z'): + tooling('prepare_release')['prepare'](version, root, run=run) + + +def test_version_error_explains_all_supported_stages(release_repo): + root, _ = release_repo + with pytest.raises(ValueError) as error: + tooling('prepare_release')['prepare']('unsupported', root) + assert all(stage in str(error.value) for stage in ('a.N', 'b.N', 'rc.N')) + + +def test_github_no_gpg_keys_response_does_not_prevent_ssh_verification(release_repo, monkeypatch): + root, git = release_repo + key = root / 'fixture-key' + subprocess.run(['ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(key)], check=True) + git( + '-c', + 'gpg.format=ssh', + '-c', + f'user.signingkey={key}', + 'tag', + '-s', + 'v2.13.5', + '-m', + 'Fixture', + ) + public_key = key.with_suffix('.pub').read_text() + real_run, real_which = subprocess.run, shutil.which + + def run(args, **kwargs): + if args[0] == 'gpg': + if kwargs.get('check'): + raise subprocess.CalledProcessError(2, args) + return subprocess.CompletedProcess(args, 2) + return real_run(args, **kwargs) + + def fetch(url): + if '/ssh_signing_keys?' in url: + return json.dumps([{'key': public_key.strip()}]) + return ( + "-----BEGIN PGP PUBLIC KEY BLOCK-----\nNote: This user hasn't uploaded any GPG keys.\n" + ) + + monkeypatch.setattr(subprocess, 'run', run) + monkeypatch.setattr(shutil, 'which', lambda name: 'gpg' if name == 'gpg' else real_which(name)) + assert tooling('verify_release_tag')['verify']( + 'v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch + ) + + +def alerts(dependabot=None, code=None, secrets=None): + def api(endpoint): + if '/dependabot/' in endpoint: + return dependabot or [] + if '/code-scanning/' in endpoint: + return code or [] + if '/secret-scanning/' in endpoint: + return secrets or [] + raise AssertionError('Unexpected endpoint') + + return api + + +def test_security_gate_accepts_empty_sources(): + assert tooling('release_security')['check']('owner/repo', api=alerts()) == [] + + +@pytest.mark.parametrize('severity', ['medium', 'high', 'critical']) +def test_security_gate_blocks_fixable_dependabot_alerts(severity): + source = { + 'security_advisory': {'severity': severity}, + 'security_vulnerability': {'first_patched_version': {'identifier': '9.0'}}, + } + assert tooling('release_security')['check']('owner/repo', api=alerts(dependabot=[source])) + + +def test_security_gate_does_not_block_low_or_unpatched_dependabot_alerts(): + sources = [ + { + 'security_advisory': {'severity': 'low'}, + 'security_vulnerability': {'first_patched_version': {'identifier': '9.0'}}, + }, + { + 'security_advisory': {'severity': 'high'}, + 'security_vulnerability': {'first_patched_version': None}, + }, + ] + assert tooling('release_security')['check']('owner/repo', api=alerts(dependabot=sources)) == [] + + +@pytest.mark.parametrize('severity', ['medium', 'high', 'critical']) +def test_security_gate_blocks_code_scanning_alerts(severity): + source = {'tool': {'name': 'CodeQL'}, 'rule': {'security_severity_level': severity}} + assert tooling('release_security')['check']('owner/repo', api=alerts(code=[source])) + + +def test_security_gate_applies_the_codeql_policy_to_codeql_only(): + source = {'tool': {'name': 'Other scanner'}, 'rule': {'security_severity_level': 'high'}} + assert tooling('release_security')['check']('owner/repo', api=alerts(code=[source])) == [] + + +def test_security_gate_requests_secret_metadata_without_secret_values(): + endpoints = [] + + def api(endpoint): + endpoints.append(endpoint) + return [] + + assert tooling('release_security')['check']('owner/repo', api=api) == [] + assert any('/secret-scanning/' in url and 'hide_secret=true' in url for url in endpoints) + + +def test_release_security_credentials_have_a_protected_environment_boundary(): + workflow = yaml.load((ROOT / '.github/workflows/cd.yml').read_text(), Loader=yaml.BaseLoader) + gates = workflow['jobs']['release-gates'] + assert gates.get('environment') == 'release-security' + assert gates['permissions']['contents'] == 'read' + assert 'id-token' not in gates['permissions'] + for name in ('publish-pypi', 'github-release'): + assert "github.event_name == 'push'" in workflow['jobs'][name]['if'] + + +def test_supported_prerelease_tags_retain_the_pypi_publish_path(): + workflow = yaml.load((ROOT / '.github/workflows/cd.yml').read_text(), Loader=yaml.BaseLoader) + condition = workflow['jobs']['publish-pypi']['if'] + assert condition == "github.event_name == 'push'" + + +def test_release_workflow_restores_the_origin_tag_object(release_repo): + root, git = release_repo + origin = root / 'origin.git' + git('init', '--bare', str(origin)) + git('remote', 'add', 'origin', str(origin)) + tag = 'v2.13.5' + git('tag', '-a', tag, '-m', 'Annotated fixture') + tag_object = git('rev-parse', tag).stdout.strip() + git('push', 'origin', f'refs/tags/{tag}') + # Reproduce checkout's replacement of the tag object with the peeled commit. + git('update-ref', f'refs/tags/{tag}', git('rev-parse', 'HEAD').stdout.strip()) + assert git('cat-file', '-t', tag).stdout.strip() == 'commit' + + workflow = yaml.load((ROOT / '.github/workflows/cd.yml').read_text(), Loader=yaml.BaseLoader) + steps = workflow['jobs']['release-gates']['steps'] + for step in steps: + if step.get('name') == 'Verify maintainer tag signature': + break + if step.get('name') == 'Restore annotated release tag': + assert step['if'] == "github.event_name == 'push'" + assert step['env']['TAG'] == '${{ github.ref_name }}' + command = [arg.replace('$TAG', tag) for arg in shlex.split(step['run'])] + subprocess.run(command, cwd=root, check=True, capture_output=True) + assert git('rev-parse', tag).stdout.strip() == tag_object + assert git('cat-file', '-t', tag).stdout.strip() == 'tag' + + +def test_release_preflight_retains_safe_actionable_failure_reasons(): + module = tooling('release_preflight') + hold = module['ReleaseHold']('Current master CI is not successful') + assert 'Current master CI is not successful' in module['failure_reason'](hold) + + +def test_security_gate_blocks_open_secret_scanning_alerts(): + assert tooling('release_security')['check']('owner/repo', api=alerts(secrets=[{}])) + + +def test_security_gate_fails_closed_when_a_source_is_unreadable(): + def denied(endpoint): + raise RuntimeError('Access denied') + + assert len(tooling('release_security')['check']('owner/repo', api=denied)) == 3 + + +def test_security_gate_fails_closed_on_malformed_response(): + assert tooling('release_security')['check']('owner/repo', api=lambda endpoint: {}) + + +@pytest.mark.parametrize( + 'value, expected', [('', False), ('false', False), ('true', True), ('on', True)] +) +def test_signed_tag_enforcement_rollout(value, expected): + assert tooling('verify_release_tag')['enforcing'](value) is expected + + +def test_signed_tag_enforcement_rejects_a_misspelled_setting(): + with pytest.raises(ValueError): + tooling('verify_release_tag')['enforcing']('tru') + + +def test_unsigned_tag_is_warning_only_until_enrolled(release_repo): + root, git = release_repo + git('tag', '-a', 'v2.13.5', '-m', 'Unsigned fixture') + verify = tooling('verify_release_tag')['verify'] + assert verify('v2.13.5', root, enforce=False, maintainers=['example'], fetch=lambda url: '') + assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=lambda url: '') + + +@pytest.mark.parametrize('signing_key_enrolled', [False, True]) +def test_ssh_signed_tag_requires_an_enrolled_signing_key( + release_repo, tmp_path, monkeypatch, signing_key_enrolled +): + root, git = release_repo + # Generated fixture material stays in pytest's temporary directory. + key = tmp_path / 'fixture-signing-key' + subprocess.run(['ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(key)], check=True) + git( + '-c', + 'gpg.format=ssh', + '-c', + f'user.signingkey={key}', + 'tag', + '-s', + 'v2.13.5', + '-m', + 'Fixture', + ) + verify = tooling('verify_release_tag')['verify'] + public_key = key.with_suffix('.pub').read_text() + + # Key files are untracked fixtures; signature verification itself does not mutate the repo. + def fetch(url): + if url.endswith('.keys'): + return '' if signing_key_enrolled else public_key + if '/ssh_signing_keys?' in url: + return json.dumps([{'key': public_key.strip()}] if signing_key_enrolled else []) + return '' + + assert ( + verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch) + is signing_key_enrolled + ) + + def no_keys(url): + return '[]' if '/ssh_signing_keys?' in url else '' + + assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=no_keys) + + # A pre-existing local/global signer configuration cannot expand enrollment. + external = root / 'external-signers' + external.write_text(f'external {public_key}') + git('config', 'gpg.ssh.allowedSignersFile', str(external)) + assert not verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=no_keys) + + if signing_key_enrolled: + monkeypatch.setenv('GITHUB_SHA', git('rev-parse', 'HEAD').stdout.strip()) + assert verify('v2.13.5', root, enforce=True, maintainers=['example'], fetch=fetch) + workflow_commit = git('commit-tree', 'HEAD^{tree}', '-m', 'Workflow fixture').stdout.strip() + monkeypatch.setenv('GITHUB_SHA', workflow_commit) + for enforce in (False, True): + assert not verify( + 'v2.13.5', root, enforce=enforce, maintainers=['example'], fetch=fetch + ) + monkeypatch.delenv('GITHUB_SHA') + # A valid signature on a different commit cannot authorize this checkout. + git('commit', '--allow-empty', '-m', 'Different workflow commit') + for enforce in (False, True): + assert not verify( + 'v2.13.5', root, enforce=enforce, maintainers=['example'], fetch=fetch + ) + + +@pytest.mark.parametrize( + 'value', [None, {}, {'kind': 'Fixed'}, {'kind': 'Unknown', 'body': 'Entry.'}] +) +def test_changie_fragment_validation_rejects_incomplete_entries(value): + with pytest.raises(ValueError): + tooling('check_changelog')['validate_fragment'](value) + + +def test_changie_fragment_validation_accepts_a_complete_entry(): + tooling('check_changelog')['validate_fragment']( + {'kind': 'Fixed', 'body': 'Correct a release check.', 'time': '2026-11-17T00:00:00Z'} + ) + + +@pytest.mark.parametrize('has_fragment', [False, True]) +@pytest.mark.parametrize('base_diff', [None, 'A\t.changes/v2.13.6.md\nM\tCHANGELOG.md\n']) +def test_changelog_check_batches_only_when_fragments_exist( + tmp_path, monkeypatch, has_fragment, base_diff +): + unreleased = tmp_path / '.changes/unreleased' + unreleased.mkdir(parents=True) + (unreleased / '.gitkeep').touch() + if has_fragment: + (unreleased / 'entry.yaml').write_text( + 'kind: Fixed\nbody: Correct a release check.\ntime: 2026-11-17T00:00:00Z\n' + ) + changelog = '## [2.13.6]\n' + (tmp_path / 'CHANGELOG.md').write_text(changelog) + calls = [] + + def run(args, **kwargs): + calls.append(args) + if 'batch' in args and not has_fragment: + raise subprocess.CalledProcessError(1, args, stderr='no unreleased changes found') + output = base_diff if args[:2] == ['git', 'diff'] else changelog + return subprocess.CompletedProcess(args, 0, output) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + sys, 'argv', ['check_changelog', *(['--base', 'base'] if base_diff else [])] + ) + monkeypatch.setattr(subprocess, 'run', run) + tooling('check_changelog')['main']() + assert any('batch' in call for call in calls) is has_fragment + + +@pytest.mark.parametrize( + 'status, conclusion, branch, event', + [ + ('completed', 'failure', 'master', 'push'), + ('in_progress', None, 'master', 'push'), + ('completed', 'success', 'feature', 'push'), + ('completed', 'success', 'master', 'pull_request'), + ], +) +def test_release_preflight_rejects_red_pending_or_unrelated_ci( + monkeypatch, status, conclusion, branch, event +): + function = tooling('release_preflight')['require_green_ci'] + response = { + 'workflow_runs': [ + { + 'head_sha': 'abc', + 'head_branch': branch, + 'event': event, + 'status': status, + 'conclusion': conclusion, + } + ] + } + monkeypatch.setattr( + subprocess, + 'run', + lambda args, **kwargs: subprocess.CompletedProcess(args, 0, json.dumps(response)), + ) + with pytest.raises(RuntimeError, match='successfully'): + function('owner/repo', 'abc') + + +def test_release_preflight_requires_the_latest_master_run_to_pass(monkeypatch): + function = tooling('release_preflight')['require_green_ci'] + response = { + 'workflow_runs': [ + { + 'head_sha': 'abc', + 'head_branch': 'master', + 'event': 'push', + 'status': 'completed', + 'conclusion': 'failure', + }, + { + 'head_sha': 'abc', + 'head_branch': 'master', + 'event': 'push', + 'status': 'completed', + 'conclusion': 'success', + }, + ] + } + monkeypatch.setattr( + subprocess, + 'run', + lambda args, **kwargs: subprocess.CompletedProcess(args, 0, json.dumps(response)), + ) + with pytest.raises(RuntimeError): + function('owner/repo', 'abc') + + +def test_release_preflight_accepts_successful_master_ci(monkeypatch): + function = tooling('release_preflight')['require_green_ci'] + response = { + 'workflow_runs': [ + { + 'head_sha': 'abc', + 'head_branch': 'master', + 'event': 'push', + 'status': 'completed', + 'conclusion': 'success', + } + ] + } + monkeypatch.setattr( + subprocess, + 'run', + lambda args, **kwargs: subprocess.CompletedProcess(args, 0, json.dumps(response)), + ) + function('owner/repo', 'abc') + + +@pytest.mark.parametrize('merged', [False, True]) +def test_release_preflight_requires_tagged_commit_on_master(release_repo, monkeypatch, merged): + root, git = release_repo + base = git('rev-parse', 'HEAD').stdout.strip() + git('update-ref', 'refs/remotes/origin/master', base) + if not merged: + git('switch', '-c', 'unmerged-release') + git('commit', '--allow-empty', '-m', 'Unmerged fixture') + git('tag', '-a', 'v2.13.5', '-m', 'Fixture') + function = tooling('release_preflight')['main'] + function.__globals__['require_green_ci'] = lambda repo, sha: None + real_run = subprocess.run + + def run(args, **kwargs): + if args[:2] == ['git', 'fetch']: + return subprocess.CompletedProcess(args, 0) + return real_run(args, **kwargs) + + monkeypatch.setattr(subprocess, 'run', run) + monkeypatch.chdir(root) + monkeypatch.setenv('GITHUB_REPOSITORY', 'Backblaze/b2-sdk-python') + monkeypatch.setenv('GITHUB_EVENT_NAME', 'push') + monkeypatch.setenv('GITHUB_REF_TYPE', 'tag') + monkeypatch.setenv('GITHUB_REF_NAME', 'v2.13.5') + if merged: + function() + else: + with pytest.raises(subprocess.CalledProcessError): + function() + + +def test_dispatch_cannot_enable_publishing(release_repo, monkeypatch): + root, git = release_repo + git('update-ref', 'refs/remotes/origin/master', git('rev-parse', 'HEAD').stdout.strip()) + function = tooling('release_preflight')['main'] + function.__globals__['require_green_ci'] = lambda repo, sha: None + real_run = subprocess.run + + def run(args, **kwargs): + if args[:2] == ['git', 'fetch']: + return subprocess.CompletedProcess(args, 0) + return real_run(args, **kwargs) + + monkeypatch.setattr(subprocess, 'run', run) + monkeypatch.chdir(root) + monkeypatch.setenv('GITHUB_REPOSITORY', 'Backblaze/b2-sdk-python') + monkeypatch.setenv('GITHUB_EVENT_NAME', 'workflow_dispatch') + monkeypatch.setenv('INPUT_DRY_RUN', 'false') + with pytest.raises(RuntimeError, match='never publishes'): + function() + + +def test_release_notes_extracts_exact_version_without_running_content(tmp_path): + changelog = tmp_path / 'CHANGELOG.md' + changelog.write_text('## [2.13.5] - today\n\n### Fixed\n\n- Entry.\n\n## [2.13.4]\n\n- Old.\n') + result = subprocess.run( + [ + 'awk', + '-v', + 'version=2.13.5', + '-f', + str(ROOT / 'scripts/release-notes.awk'), + str(changelog), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert 'Entry.' in result.stdout + assert 'Old.' not in result.stdout + + +def test_release_notes_fails_when_version_is_missing(tmp_path): + changelog = tmp_path / 'CHANGELOG.md' + changelog.write_text('## [2.13.4]\n') + result = subprocess.run( + [ + 'awk', + '-v', + 'version=2.13.5', + '-f', + str(ROOT / 'scripts/release-notes.awk'), + str(changelog), + ], + capture_output=True, + text=True, + ) + assert result.returncode != 0 diff --git a/uv.lock b/uv.lock index 46673c7fb..37f1c235c 100644 --- a/uv.lock +++ b/uv.lock @@ -103,7 +103,7 @@ nox = [ { name = "pip" }, ] release = [ - { name = "towncrier" }, + { name = "pyyaml" }, ] test = [ { name = "coverage" }, @@ -118,6 +118,7 @@ test = [ { name = "pytest-watcher" }, { name = "pytest-xdist" }, { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, + { name = "pyyaml" }, { name = "responses" }, { name = "tenacity" }, { name = "tqdm" }, @@ -159,7 +160,7 @@ nox = [ { name = "nox", specifier = ">=2025.11.12" }, { name = "pip", specifier = ">=26.2.2" }, ] -release = [{ name = "towncrier", specifier = "==23.11.0" }] +release = [{ name = "pyyaml", specifier = ">=6.0.3,<7" }] test = [ { name = "coverage", specifier = ">=7.13.5,<8" }, { name = "pip", specifier = ">=26.2.2" }, @@ -173,6 +174,7 @@ test = [ { name = "pytest-watcher", specifier = ">=0.6.3,<1" }, { name = "pytest-xdist", specifier = ">=3.8.0,<4" }, { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'", specifier = ">=311,<312" }, + { name = "pyyaml", specifier = ">=6.0.3,<7" }, { name = "responses", specifier = ">=0.26.0,<1" }, { name = "tenacity", specifier = ">=9.1.4,<10" }, { name = "tqdm", specifier = ">=4.5.0,<5.0.0" }, @@ -598,19 +600,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", size = 8769, upload-time = "2022-07-01T12:21:02.467Z" }, ] -[[package]] -name = "incremental" -version = "24.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/3c/82e84109e02c492f382c711c58a3dd91badda6d746def81a1465f74dc9f5/incremental-24.11.0.tar.gz", hash = "sha256:87d3480dbb083c1d736222511a8cf380012a8176c2456d01ef483242abbbcf8c", size = 24000, upload-time = "2025-11-28T02:30:17.861Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/55/0f4df2a44053867ea9cbea73fc588b03c55605cd695cee0a3d86f0029cb2/incremental-24.11.0-py3-none-any.whl", hash = "sha256:a34450716b1c4341fe6676a0598e88a39e04189f4dce5dc96f656e040baa10b3", size = 21109, upload-time = "2025-11-28T02:30:16.442Z" }, -] - [[package]] name = "iniconfig" version = "2.3.0" @@ -1765,21 +1754,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/57/4a85c617b6349962320b4afbe1386d5b05fd57218ee8ad6e82e0110ec9ee/tornado-6.6-cp315-cp315t-win_arm64.whl", hash = "sha256:7de62e2dfdf559fd7154d2b9aa58ff3d9cff8304ae0519341feb082618bc2a33", size = 530780, upload-time = "2026-10-21T15:31:03.425Z" }, ] -[[package]] -name = "towncrier" -version = "23.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "incremental" }, - { name = "jinja2" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e0/43/bb5161b32c538a099e1e1a8b79eedd4556dca925facc3b11e0ea7ab96d40/towncrier-23.11.0.tar.gz", hash = "sha256:13937c247e3f8ae20ac44d895cf5f96a60ad46cfdcc1671759530d7837d9ee5d", size = 51483, upload-time = "2023-11-08T14:27:30.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/36/5a/6546c8ea2bee64c9e9552e06fdb31118a4f23a12aa161581b5fefd070152/towncrier-23.11.0-py3-none-any.whl", hash = "sha256:2e519ca619426d189e3c98c99558fe8be50c9ced13ea1fc20a4a353a95d2ded7", size = 49327, upload-time = "2023-11-08T14:27:28.45Z" }, -] - [[package]] name = "tqdm" version = "4.67.1"