From 8504f52beba46efdf5f3fc7888002d0245ee579a Mon Sep 17 00:00:00 2001 From: scarreras Date: Tue, 17 Nov 2026 23:53:43 -0300 Subject: [PATCH] test: add regression tests for known defects (tests only) Add regression tests for known defects. Tests of defects that are still present are strict expected failures (`xfail(strict=True)`, reason "Known defect C: ..."), tagged with a new `known_defect` marker, so the default run stays green and fails as soon as a defect is fixed, which prompts removing the decorator. `pytest --runxfail -m known_defect --api=v3 test/unit test/static` runs them as real failures. Tests that pass today run in the normal suite. No production code changes. Test wiring: `test/known_defect.py` (decorator) and marker registration in `test/conftest.py`. Co-Authored-By: Claude Sonnet 5.5 --- ...sion-tests-known-defects.infrastructure.md | 1 + test/conftest.py | 8 + test/known_defect.py | 40 ++ test/static/test_regression_repo_checks.py | 101 +++++ .../account_info/test_regression_sqlite.py | 84 ++++ test/unit/bucket/test_regression_bucket.py | 383 ++++++++++++++++++ test/unit/stream/test_regression_streams.py | 193 +++++++++ .../unit/sync/test_regression_sync_actions.py | 162 ++++++++ test/unit/test_regression_misc.py | 124 ++++++ test/unit/v_all/test_regression_legacy_api.py | 62 +++ 10 files changed, 1158 insertions(+) create mode 100644 changelog.d/+regression-tests-known-defects.infrastructure.md create mode 100644 test/known_defect.py create mode 100644 test/static/test_regression_repo_checks.py create mode 100644 test/unit/account_info/test_regression_sqlite.py create mode 100644 test/unit/bucket/test_regression_bucket.py create mode 100644 test/unit/stream/test_regression_streams.py create mode 100644 test/unit/sync/test_regression_sync_actions.py create mode 100644 test/unit/test_regression_misc.py create mode 100644 test/unit/v_all/test_regression_legacy_api.py diff --git a/changelog.d/+regression-tests-known-defects.infrastructure.md b/changelog.d/+regression-tests-known-defects.infrastructure.md new file mode 100644 index 000000000..4cffb7e48 --- /dev/null +++ b/changelog.d/+regression-tests-known-defects.infrastructure.md @@ -0,0 +1 @@ +Add regression tests for known defects; tests of defects that are still present are strict expected failures marked `known_defect` (run them with `pytest --runxfail -m known_defect`). diff --git a/test/conftest.py b/test/conftest.py index 489730dd6..cd17e2a42 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -14,6 +14,14 @@ pytest_plugins = ['b2sdk.v3.testing'] +def pytest_configure(config): + config.addinivalue_line( + 'markers', + 'known_defect: test of a defect that is still present, marked as a strict expected failure ' + '(see test/known_defect.py)', + ) + + @pytest.fixture def bg_executor(): with concurrent.futures.ThreadPoolExecutor() as executor: diff --git a/test/known_defect.py b/test/known_defect.py new file mode 100644 index 000000000..744f90b98 --- /dev/null +++ b/test/known_defect.py @@ -0,0 +1,40 @@ +###################################################################### +# +# File: test/known_defect.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations + +import pytest + + +def known_defect(defect_id: str, summary: str, *, condition: bool = True): + """ + Mark a test that asserts the correct behaviour of a defect that is still present. + + The test is a strict expected failure (``xfail(strict=True)``), so the default test run stays + green, and the run fails as soon as the defect is fixed ("XPASS(strict)"). When that happens, + delete the ``@known_defect(...)`` line: the test then becomes a normal regression test. + + The tests carry the ``known_defect`` marker, so they can be run (and are reported as real + failures) on demand with one command, for example:: + + pytest --runxfail -m known_defect --api=v3 test/unit test/static + + :param defect_id: identifier of the defect, for example ``'C16'`` + :param summary: one-line description of the defect + :param condition: when false the defect does not apply (for example to another apiver) and the + test is a normal test + """ + + def decorator(func): + if not condition: + return func + func = pytest.mark.known_defect(func) + return pytest.mark.xfail(strict=True, reason=f'Known defect {defect_id}: {summary}')(func) + + return decorator diff --git a/test/static/test_regression_repo_checks.py b/test/static/test_regression_repo_checks.py new file mode 100644 index 000000000..a7f51b07e --- /dev/null +++ b/test/static/test_regression_repo_checks.py @@ -0,0 +1,101 @@ +###################################################################### +# +# File: test/static/test_regression_repo_checks.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Static regression checks of repository files (docs, changelog, packaging, sources). + +They read repository files only and are skipped when the repository files are not available. + +Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct +behaviour and are strict expected failures, so the default run stays green and fails with +"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run +``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C`` identifiers are +short labels that also appear in the test names and decorators. +""" + +from __future__ import annotations + +import re +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from test.known_defect import known_defect + +ROOT = Path(__file__).resolve().parents[2] + + +def _read(rel: str) -> str: + path = ROOT / rel + if not path.is_file(): + pytest.skip(f'{rel} is not available (not a full repository checkout)') + return path.read_text(encoding='utf-8') + + +@known_defect('C30', 'typo in .github/SUPPORT.md') +def test_c30_support_file_has_no_known_typo(): + """C30: typo 'Issuew' in .github/SUPPORT.md (codespell did not catch it).""" + assert 'Issuew' not in _read('.github/SUPPORT.md'), 'typo is still present' + + +@known_defect('C33', 'Bucket.upload leaves a FIXME about min_part_size semantics') +def test_c33_min_part_size_fixme_resolved(): + """C33: min_part_size docs/code mismatch must not be left as a FIXME in Bucket.upload.""" + assert 'FIXME: Bucket.upload documents wrong logic' not in _read( + 'b2sdk/_internal/bucket.py' + ), 'the FIXME is still present' + + +@known_defect('C59', 'the README does not mention the "full" install extra') +def test_c59_readme_mentions_full_extra(): + """C59: README must mention the `full` install extra (tqdm/pydantic support).""" + assert re.search(r'b2sdk\[full\]', _read('README.md')), 'README does not mention b2sdk[full]' + + +@known_defect('C4', 'the Quick Start uses a class and an argument name that do not exist') +def test_c4_quick_start_uses_existing_names(): + """C4: Quick Start must not use the non-existent class EncryptionSettings or the wrong `id=` argument.""" + text = _read('doc/source/quick_start.rst') + assert 'EncryptionSettings(' not in text, 'EncryptionSettings does not exist' + assert not re.search(r'EncryptionKey\([^)]*\bid=', text), 'EncryptionKey has no id= argument' + + +@known_defect( + 'C4', 'the development-version install instructions install from PyPI, not the checkout' +) +def test_c4_install_doc_development_version_installs_from_checkout(): + """C4: 'Installing a development version' must install the checkout, not PyPI.""" + section = _read('doc/source/install.rst').split('Installing a development version', 1)[1] + assert re.search( + r'pip install (-e )?[.\w/]*\.(\s|$)|pip install -e', section + ), 'development-version section installs from PyPI' + + +@known_defect('C36', 'importing b2sdk fails when the package metadata is not installed') +def test_c36_version_import_works_when_package_metadata_is_missing(tmp_path): + """C36: importing a vendored/uninstalled copy of b2sdk must not fail for missing dist metadata.""" + script = textwrap.dedent( + f""" + import importlib.metadata as m, sys + orig = m.version + def fake(name): + if name == 'b2sdk': + raise m.PackageNotFoundError(name) + return orig(name) + m.version = fake + sys.path.insert(0, {str(ROOT)!r}) + import b2sdk.version + """ + ) + result = subprocess.run( + [sys.executable, '-c', script], capture_output=True, text=True, cwd=tmp_path + ) + assert result.returncode == 0, result.stderr[-300:] diff --git a/test/unit/account_info/test_regression_sqlite.py b/test/unit/account_info/test_regression_sqlite.py new file mode 100644 index 000000000..efdf29b4e --- /dev/null +++ b/test/unit/account_info/test_regression_sqlite.py @@ -0,0 +1,84 @@ +###################################################################### +# +# File: test/unit/account_info/test_regression_sqlite.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Regression tests for SqliteAccountInfo. + +Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct +behaviour and are strict expected failures, so the default run stays green and fails with +"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run +``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C`` identifiers are +short labels that also appear in the test names and decorators. +""" + +from __future__ import annotations + +import os +import threading + +import pytest +from apiver_deps import ( + B2_ACCOUNT_INFO_ENV_VAR, + XDG_CONFIG_HOME_ENV_VAR, + SqliteAccountInfo, +) + +from test.known_defect import known_defect + +pytestmark = [pytest.mark.apiver(from_ver=2)] + + +@pytest.fixture +def home(monkeypatch, tmp_path): + monkeypatch.setenv('HOME', str(tmp_path)) + monkeypatch.setenv('USERPROFILE', str(tmp_path)) + monkeypatch.delenv(B2_ACCOUNT_INFO_ENV_VAR, raising=False) + monkeypatch.delenv(XDG_CONFIG_HOME_ENV_VAR, raising=False) + return tmp_path + + +class TestProfilePaths: + @known_defect( + 'C47', + 'an existing ~/.b2db-.sqlite is ignored because "~" is not expanded before the existence check', + ) + def test_c47_existing_legacy_profile_database_is_used_with_xdg(self, home, monkeypatch): + """C47: an existing ~/.b2db-.sqlite must be found even when XDG_CONFIG_HOME is set.""" + monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, str(home / 'xdg')) + legacy = home / '.b2db-foo.sqlite' + legacy.write_bytes(b'') + path = SqliteAccountInfo.get_user_account_info_path(profile='foo') + assert os.path.realpath(path) == os.path.realpath(str(legacy)), f'got {path}' + + +class TestThreadConnections: + @known_defect('C37', "close() closes only the calling thread's database connection") + def test_c37_close_closes_connections_of_other_threads(self, tmp_path): + """C37: close() must close the database connections opened by other threads, too.""" + info = SqliteAccountInfo(file_name=str(tmp_path / 'acct.sqlite')) + opened, closed_by_main, result = threading.Event(), threading.Event(), {} + + def worker(): + conn = info._get_connection() # connection of this worker thread + opened.set() + assert closed_by_main.wait(timeout=10), 'main thread did not call close()' + try: + conn.execute('select 1') + result['closed'] = False + except Exception: # sqlite3.ProgrammingError: Cannot operate on a closed database + result['closed'] = True + + thread = threading.Thread(target=worker) + thread.start() + assert opened.wait(timeout=10), 'worker thread did not open a connection' + info.close() + closed_by_main.set() + thread.join(timeout=10) + assert ( + result.get('closed') is True + ), 'connection of another thread is still open after close()' diff --git a/test/unit/bucket/test_regression_bucket.py b/test/unit/bucket/test_regression_bucket.py new file mode 100644 index 000000000..09f881028 --- /dev/null +++ b/test/unit/bucket/test_regression_bucket.py @@ -0,0 +1,383 @@ +###################################################################### +# +# File: test/unit/bucket/test_regression_bucket.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Regression tests for Bucket and B2Api behaviour on the RawSimulator. + +Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct +behaviour and are strict expected failures, so the default run stays green and fails with +"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run +``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C`` identifiers are +short labels that also appear in the test names and decorators. +""" + +from __future__ import annotations + +import datetime +import json +import locale + +import apiver_deps +import pytest +import requests +import responses +from apiver_deps import ( + B2Api, + B2Http, + B2HttpApiConfig, + B2RawHTTPApi, + Filter, + InMemoryAccountInfo, + InMemoryCache, + RawSimulator, +) +from apiver_deps_exception import NonExistentBucket + +from test.known_defect import known_defect + +from .test_bucket import TestCaseWithBucket + +pytestmark = [pytest.mark.apiver(from_ver=2)] + + +class TestRegressionCopyAndLs(TestCaseWithBucket): + @known_defect( + 'C1', 'copy() with expires/content_disposition/cache_control but no content_type is refused' + ) + def test_c1_copy_with_expiry_but_no_content_type(self): + """C1: copy() with expires/headers but no content_type must work (2.13.3 notes).""" + file_id = self.bucket.upload_bytes(b'hello world', 'src.txt').id_ + copied = self.bucket.copy( + file_id, + 'copied.txt', + expires='Thu, 01 Dec 2050 16:00:00 GMT', + content_disposition='attachment', + cache_control='max-age=5', + ) + assert copied.file_info['b2-expires'] == 'Thu, 01 Dec 2050 16:00:00 GMT', copied.file_info + assert copied.file_info['b2-content-disposition'] == 'attachment', copied.file_info + + @known_defect( + 'C5', + 'ls("x") stops at an object named "x" and never lists "x/..."', + condition=apiver_deps.V >= 3, + ) + def test_c5_ls_path_without_slash_lists_folder_next_to_same_named_file(self): + """C5: ls('x') with an object 'x' and objects 'x/...' must also list 'x/...'.""" + self.bucket.upload_bytes(b'data', 'x') + self.bucket.upload_bytes(b'data', 'x/1.txt') + names = [fv.file_name for fv, _ in self.bucket.ls('x', recursive=True)] + assert 'x/1.txt' in names, names + + @known_defect( + 'C5', + 'B2Folder for a prefix naming a single object hits an AssertionError', + condition=apiver_deps.V >= 3, + ) + def test_c5_folder_sync_source_with_prefix_naming_a_single_object(self): + """C5: B2Folder for 'b2://bucket/x' must not hit an AssertionError when only 'x' exists.""" + from apiver_deps import B2Folder + + self.bucket.upload_bytes(b'data', 'x') + folder = B2Folder(self.bucket_name, 'x', self.api) + # must not raise AssertionError (it may list nothing or the object, but not crash) + list(folder.all_files(None)) + + @known_defect( + 'C6', 'ls() of an exact file name ignores the filters', condition=apiver_deps.V >= 3 + ) + def test_c6_exact_filename_respects_exclude_filter(self): + """C6: an exclude filter naming the exact file must hide it from ls().""" + self.bucket.upload_bytes(b'hello world', 'b/a.txt') + result = [ + fv.file_name + for fv, _ in self.bucket.ls( + 'b/a.txt', recursive=True, filters=[Filter.exclude('b/a.txt')] + ) + ] + assert result == [], 'the filter was ignored' + + @known_defect( + 'C6', 'ls() of an exact file name ignores the filters', condition=apiver_deps.V >= 3 + ) + def test_c6_exact_filename_respects_include_filter_for_other_name(self): + """C6: an include filter not matching the exact file must hide it from ls().""" + self.bucket.upload_bytes(b'hello world', 'b/a.txt') + result = [ + fv.file_name + for fv, _ in self.bucket.ls( + 'b/a.txt', recursive=True, filters=[Filter.include('b/z.txt')] + ) + ] + assert result == [], 'the filter was ignored' + + +class TestRegressionUploadOptions(TestCaseWithBucket): + @known_defect('C45', 'multipart uploads ignore custom_upload_timestamp') + def test_c45_custom_timestamp_is_kept_on_large_upload(self): + """C45: a custom upload timestamp must be applied to multi-part uploads too.""" + data = self._make_data(self.simulator.MIN_PART_SIZE * 3) + file_version = self.bucket.upload_bytes(data, 'large', custom_upload_timestamp=12345) + assert file_version.upload_timestamp == 12345, file_version.upload_timestamp + + @known_defect('C45', 'multipart uploads ignore custom_upload_timestamp') + def test_c45_custom_timestamp_is_kept_on_large_unfinished_file(self): + """C45: the started large file carries the custom timestamp (visible after the upload).""" + data = self._make_data(self.simulator.MIN_PART_SIZE * 3) + self.bucket.upload_bytes(data, 'large2', custom_upload_timestamp=777) + stored = self.bucket.get_file_info_by_name('large2') + assert stored.upload_timestamp == 777, stored.upload_timestamp + + @pytest.mark.skipif( + not hasattr(locale, 'setlocale'), reason='locale support is required for this test' + ) + @known_defect('C32', 'the "expires" header uses day and month names of the process locale') + def test_c32_expires_header_uses_english_names_under_foreign_locale(self): + """C32: the 'expires' date must not use process-locale day/month names.""" + old = locale.setlocale(locale.LC_TIME) + for name in ('de_DE.UTF-8', 'de_DE.utf8', 'fr_FR.UTF-8', 'es_ES.UTF-8'): + try: + locale.setlocale(locale.LC_TIME, name) + break + except locale.Error: + continue + else: + pytest.skip('no non-English locale available on this host') + try: + expires = datetime.datetime(2050, 12, 1, 16, 0, tzinfo=datetime.timezone.utc) + file_version = self.bucket.upload_bytes(b'x', 'f.txt', expires=expires) + finally: + locale.setlocale(locale.LC_TIME, old) + assert ( + file_version.file_info['b2-expires'] == 'Thu, 01 Dec 2050 16:00:00 GMT' + ), file_version.file_info + + +class TestRegressionApi: + def _api(self, account_info=None, cache=None): + api = B2Api( + account_info or InMemoryAccountInfo(), + cache=cache if cache is not None else InMemoryCache(), + api_config=B2HttpApiConfig(_raw_api_class=RawSimulator), + ) + account_id, master_key = api.session.raw_api.create_account() + api.authorize_account( + application_key_id=account_id, application_key=master_key, realm='production' + ) + return api, master_key + + @known_defect('C2', 'delete_bucket does not remove the bucket name from the name cache') + def test_c2_deleted_bucket_not_returned_from_name_cache(self): + """C2: after delete_bucket, a lookup by name must fail, not return the stale ID.""" + api, _ = self._api() + bucket = api.create_bucket('to-delete', 'allPrivate') + assert api.get_bucket_by_name('to-delete').id_ == bucket.id_, 'cache was not warm' + api.delete_bucket(bucket) + with pytest.raises(NonExistentBucket): + api.get_bucket_by_name('to-delete') + + @known_defect('C28', 'the create_bucket mismatch message prints the same name twice') + def test_c28_create_bucket_mismatch_message_names_both_buckets(self): + """C28: the assertion message must show requested and returned names, not name twice.""" + from unittest import mock + + api, _ = self._api() + wrong = mock.Mock() + wrong.name = 'returned-other-name' + wrong.type_ = 'allPrivate' + with mock.patch.object( + api.BUCKET_FACTORY_CLASS, 'from_api_bucket_dict', return_value=wrong + ): + with pytest.raises(AssertionError) as exc_info: + api.create_bucket('requested-name', 'allPrivate') + message = str(exc_info.value) + assert 'requested-name' in message, message + assert 'returned-other-name' in message, message + + +class TestRegressionRawApiContract: + @responses.activate + @known_defect( + 'C35', 'set_notification_rules fails for a generator although it is typed as Iterable' + ) + def test_c35_set_notification_rules_accepts_a_generator(self): + """C35: set_notification_rules is typed Iterable; a generator must serialize to JSON.""" + raw_api = B2RawHTTPApi( + B2Http(B2HttpApiConfig(requests.Session, install_clock_skew_hook=False)) + ) + responses.add( + responses.POST, + 'https://api.example.com/b2api/v4/b2_set_bucket_notification_rules', + json={'eventNotificationRules': []}, + ) + rule = { + 'eventTypes': ['b2:ObjectCreated:*'], + 'isEnabled': True, + 'name': 'r', + 'objectNamePrefix': '', + 'targetConfiguration': { + 'targetType': 'webhook', + 'url': 'https://example.com/hook', + 'customHeaders': [], + }, + } + sent = raw_api.set_bucket_notification_rules( + 'https://api.example.com', 'token', 'bucket-id', (r for r in [rule]) + ) + assert sent == [], sent + body = json.loads(responses.calls[0].request.body) + assert body['eventNotificationRules'] == [rule], body + + +class TestRegressionLargeUploadFailure: + @known_defect('C8', 'part uploads keep running after one part of a large upload failed') + def test_c8_failed_part_stops_remaining_part_uploads(self): + """C8: once one part fails the whole upload fails; queued parts must not keep uploading.""" + import time + from unittest import mock + + from apiver_deps_exception import AccessDenied + + api = B2Api( + InMemoryAccountInfo(), + cache=InMemoryCache(), + api_config=B2HttpApiConfig(_raw_api_class=RawSimulator), + max_upload_workers=1, + ) + simulator = api.session.raw_api + account_id, master_key = simulator.create_account() + api.authorize_account( + application_key_id=account_id, application_key=master_key, realm='production' + ) + bucket = api.create_bucket('big', 'allPrivate') + original = simulator.upload_part + started = [] + + def flaky_upload_part(upload_url, token, part_number, *args, **kwargs): + started.append(part_number) + if part_number == 1: + raise AccessDenied() + time.sleep(0.2) + return original(upload_url, token, part_number, *args, **kwargs) + + data = b'x' * (simulator.MIN_PART_SIZE * 6) + with mock.patch.object(simulator, 'upload_part', flaky_upload_part): + with pytest.raises(Exception): + bucket.upload_bytes(data, 'big.bin') + started_at_failure = len(started) + time.sleep(1.5) # bounded wait for any background part threads + assert ( + len(started) <= started_at_failure + 1 + ), f'parts kept uploading after the failure: {started}' + + +class TestRegressionDownloadedFileContract: + @responses.activate + @known_defect('C56', 'DownloadedFile.save() called twice re-reads a used response') + def test_c56_downloaded_file_can_be_saved_twice_from_a_real_response(self): + """C56: save() called twice on a real streamed HTTP response must give the full data again + (or fail clearly); the simulator's in-memory response hides this.""" + import hashlib + import io + from unittest import mock + + from apiver_deps import DownloadedFile, Range, SimpleDownloader + + data = b'0123456789' * 100 + url = 'https://download.example.com/file/bucket/name' + responses.add(responses.GET, url, body=data, content_type='application/octet-stream') + response = requests.get(url, stream=True) + + download_version = mock.MagicMock() + download_version.content_length = len(data) + download_version.content_sha1 = hashlib.sha1(data).hexdigest() + download_version.content_encoding = None + download_version.range_ = Range(0, len(data) - 1) + download_version.api.api_config.decode_content = False + download_manager = mock.MagicMock() + download_manager.strategies = [SimpleDownloader(force_chunk_size=64)] + + downloaded = DownloadedFile(download_version, download_manager, None, response, None, None) + first, second = io.BytesIO(), io.BytesIO() + downloaded.save(first) + assert first.getvalue() == data, 'first save() returned wrong data' + downloaded.save(second) + assert second.getvalue() == data, 'second save() returned wrong data' + + +class TestRegressionGuards(TestCaseWithBucket): + def test_c43_guard_sse_c_key_is_not_printed_or_logged(self): + """C43: SSE-C key material must not show in repr/str or DEBUG logs of public calls.""" + import logging + + from apiver_deps import ( + EncryptionAlgorithm, + EncryptionKey, + EncryptionMode, + EncryptionSetting, + ) + + secret = b'sse-c-key-material-MUST-NOT-LEAK' + setting = EncryptionSetting( + mode=EncryptionMode.SSE_C, + algorithm=EncryptionAlgorithm.AES256, + key=EncryptionKey(secret=secret, key_id='key-id'), + ) + for text in (repr(setting), str(setting), repr(setting.key), str(setting.key)): + assert secret.decode() not in text, 'key material in printed form' + assert repr(secret) not in text, 'key material in printed form' + + records = [] + + class Collector(logging.Handler): + def emit(self, record): + records.append(record.getMessage()) + + handler = Collector(level=logging.DEBUG) + root = logging.getLogger() + old_level = root.level + root.addHandler(handler) + root.setLevel(logging.DEBUG) + try: + fv = self.bucket.upload_bytes(b'data', 'enc.txt', encryption=setting) + self.bucket.copy( + fv.id_, 'enc-copy.txt', source_encryption=setting, destination_encryption=setting + ) + finally: + root.removeHandler(handler) + root.setLevel(old_level) + joined = '\n'.join(records) + assert records, 'expected DEBUG records to be captured' + assert secret.decode() not in joined, 'key material in DEBUG logs' + assert repr(secret) not in joined, 'key material in DEBUG logs' + + def test_c13_guard_failed_part_upload_retries_with_a_fresh_upload_url(self): + """C13: after a retryable part failure the retry must not reuse the failed URL/token.""" + from unittest import mock + + from apiver_deps_exception import ServiceError # noqa: F401 (retryable class below) + + from .test_bucket import CanRetry + + attempts = [] + original = self.simulator.upload_part + + def recording(upload_url, token, part_number, *args, **kwargs): + attempts.append((part_number, upload_url, token)) + return original(upload_url, token, part_number, *args, **kwargs) + + self.simulator.set_upload_errors([CanRetry(True)]) + data = self._make_data(self.simulator.MIN_PART_SIZE * 3) + with mock.patch.object(self.simulator, 'upload_part', recording): + self.bucket.upload_bytes(data, 'large.bin') + self._check_file_contents('large.bin', data) + per_part = {} + for part_number, url, token in attempts: + per_part.setdefault(part_number, []).append((url, token)) + for part_number, tried in per_part.items(): + assert len(set(tried)) == len(tried), f'part {part_number} reused a failed URL/token' diff --git a/test/unit/stream/test_regression_streams.py b/test/unit/stream/test_regression_streams.py new file mode 100644 index 000000000..6fc21741b --- /dev/null +++ b/test/unit/stream/test_regression_streams.py @@ -0,0 +1,193 @@ +###################################################################### +# +# File: test/unit/stream/test_regression_streams.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Regression tests for stream and upload-source classes. + +Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct +behaviour and are strict expected failures, so the default run stays green and fails with +"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run +``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C`` identifiers are +short labels that also appear in the test names and decorators. +""" + +from __future__ import annotations + +import hashlib +import io + +import pytest + +from b2sdk._internal.stream.chained import ChainedStream +from b2sdk._internal.stream.range import RangeOfInputStream +from b2sdk._internal.transfer.outbound.upload_source import ( + UploadSourceLocalFileRange, + UploadSourceStream, + UploadSourceStreamRange, +) +from b2sdk._internal.utils import hex_sha1_of_unlimited_stream +from test.known_defect import known_defect + + +class ShortReadStream(io.RawIOBase): + """A pipe-like stream that never returns more than `chunk` bytes per read, before the real end.""" + + def __init__(self, data: bytes, chunk: int): + self._buf = io.BytesIO(data) + self._chunk = chunk + + def readable(self): + return True + + def read(self, size=-1): + if size is None or size < 0 or size > self._chunk: + size = self._chunk + return self._buf.read(size) + + +class ClosableTracker(io.BytesIO): + def close(self): + self.was_closed = True + super().close() + + +class TestHashingShortReads: + @pytest.mark.parametrize('chunk', [1, 7, 100]) + @known_defect( + 'C16', + 'hex_sha1_of_unlimited_stream stops at the first short read, so length and SHA1 cover only a prefix', + ) + def test_c16_hash_and_length_cover_whole_stream_with_short_reads(self, chunk): + """C16: a source returning fewer bytes than asked must not truncate hash/length.""" + data = bytes(range(256)) * 20 + sha1, length = hex_sha1_of_unlimited_stream(ShortReadStream(data, chunk)) + assert length == len(data), f'length {length} != {len(data)}' + assert sha1 == hashlib.sha1(data).hexdigest(), 'SHA1 does not cover the whole stream' + + @known_defect( + 'C16', + 'UploadSourceStream length and SHA1 cover only a prefix when the stream returns short reads', + ) + def test_c16_upload_source_stream_reports_full_length_with_short_reads(self): + """C16: UploadSourceStream computing length/sha1 from a pipe-like stream.""" + data = b'x' * 5000 + source = UploadSourceStream(lambda: ShortReadStream(data, 13)) + assert source.get_content_length() == len(data), 'length does not cover the whole stream' + assert ( + source.get_content_sha1() == hashlib.sha1(data).hexdigest() + ), 'SHA1 does not cover the whole stream' + + +class TestRangeOfInputStream: + @known_defect( + 'C46', + 'RangeOfInputStream.read(-1) passes -1 to the inner stream and reads past the end of the range', + ) + def test_c46_read_minus_one_does_not_go_past_range_end(self): + """C46: read(-1) must be limited to the range, like read() is.""" + stream = RangeOfInputStream(io.BytesIO(b'0123456789'), offset=2, length=3) + assert stream.read(-1) == b'234', 'read(-1) went past the end of the range' + + def test_c46_read_none_stays_within_range(self): + """Control for C46: read(None) is limited to the range (passes today).""" + stream = RangeOfInputStream(io.BytesIO(b'0123456789'), offset=2, length=3) + assert stream.read() == b'234', 'read() went past the end of the range' + + +class TestChainedStream: + @known_defect( + 'C11', + 'ChainedStream.read() with the default size raises TypeError instead of reading everything', + ) + def test_c11_default_read_returns_everything(self): + """C11: read() with default size reads everything (docs), not a TypeError.""" + stream = ChainedStream([lambda: io.BytesIO(b'abc'), lambda: io.BytesIO(b'def')]) + assert stream.read() == b'abcdef', 'read() did not return all data' + + @known_defect('C11', 'ChainedStream.read(None) raises TypeError instead of reading everything') + def test_c11_explicit_none_returns_everything(self): + """C11: read(None) is documented as 'read everything'.""" + stream = ChainedStream([lambda: io.BytesIO(b'abc'), lambda: io.BytesIO(b'def')]) + assert stream.read(None) == b'abcdef', 'read(None) did not return all data' + + def test_c12_guard_sized_reads_cross_stream_boundaries(self): + """C12: guard that sized reads span the chained streams and end at EOF.""" + stream = ChainedStream([lambda: io.BytesIO(b'abc'), lambda: io.BytesIO(b'def')]) + assert stream.read(4) == b'abcd', 'sized read did not cross the stream boundary' + assert stream.read(10) == b'ef', 'second sized read returned wrong data' + assert stream.read(10) == b'', 'read at EOF did not return empty bytes' + + +class TestUploadSourceStreamRange: + @known_defect('C15', 'UploadSourceStreamRange built without a length fails on the first read') + def test_c15_stream_range_without_length_can_be_read(self): + """C15: docs allow omitting the stream length; open() then read must work.""" + data = b'0123456789' + source = UploadSourceStreamRange(lambda: io.BytesIO(data), offset=4) + with source.open() as fp: + assert fp.read() == data[4:], 'wrong data read from the stream range' + + @known_defect('C15', 'UploadSourceStreamRange built without a length cannot report its length') + def test_c15_stream_range_without_length_computes_length(self): + """C15: length is computed lazily when not given.""" + source = UploadSourceStreamRange(lambda: io.BytesIO(b'0123456789'), offset=0) + assert source.get_content_length() == 10, 'wrong computed length' + + @known_defect( + 'C14', 'UploadSourceStream does not close the stream it opened to compute the checksum' + ) + def test_c14_stream_is_closed_after_checksum_computation(self): + """C14: computing the checksum must close the stream it opened.""" + opened = [] + + def opener(): + s = ClosableTracker(b'hello world') + s.was_closed = False + opened.append(s) + return s + + UploadSourceStream(opener).get_content_sha1() + assert opened, 'the source never opened a stream' + assert all(s.was_closed for s in opened), 'a stream opened for the checksum was left open' + + +class TestUploadSourceLocalFileRange: + @pytest.mark.parametrize('offset', [-3, 15]) + @known_defect( + 'C52', + 'UploadSourceLocalFileRange accepts a negative offset or an offset past the end of the file', + ) + def test_c52_invalid_offset_is_rejected(self, tmp_path, offset): + """C52: negative offset or offset past end (file is 10 bytes) must be refused.""" + path = tmp_path / 'f.bin' + path.write_bytes(b'0123456789') + with pytest.raises(ValueError): + UploadSourceLocalFileRange(str(path), offset=offset) + + +class TestUnboundWriteIntentMemory: + @known_defect( + 'C57', 'UnboundSourceBytes copies the chunk, using about twice the documented memory' + ) + def test_c57_unbound_buffer_wrapping_does_not_copy_the_chunk(self): + """C57: wrapping a bytearray chunk for upload must not allocate a second full copy.""" + import tracemalloc + + from b2sdk._internal.transfer.emerge.unbound_write_intent import UnboundSourceBytes + + size = 8 * 1024 * 1024 + chunk = bytearray(size) + tracemalloc.start() + try: + before = tracemalloc.get_traced_memory()[0] + source = UnboundSourceBytes(chunk, lambda: None) + extra = tracemalloc.get_traced_memory()[0] - before + finally: + tracemalloc.stop() + assert source.get_content_length() == size, 'wrong content length' + assert extra < size // 2, f'wrapping allocated {extra} extra bytes for a {size} byte chunk' diff --git a/test/unit/sync/test_regression_sync_actions.py b/test/unit/sync/test_regression_sync_actions.py new file mode 100644 index 000000000..bf00b8d9e --- /dev/null +++ b/test/unit/sync/test_regression_sync_actions.py @@ -0,0 +1,162 @@ +###################################################################### +# +# File: test/unit/sync/test_regression_sync_actions.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Regression tests for sync actions and downloads to local files. + +Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct +behaviour and are strict expected failures, so the default run stays green and fails with +"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run +``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C`` identifiers are +short labels that also appear in the test names and decorators. +""" + +from __future__ import annotations + +import os +from unittest import mock + +import pytest +from apiver_deps import ( + B2Api, + B2HttpApiConfig, + InMemoryAccountInfo, + InMemoryCache, + RawSimulator, + ServerDefaultSyncEncryptionSettingsProvider, +) +from apiver_deps_exception import ChecksumMismatch, TruncatedOutput + +from b2sdk._internal.scan.path import B2Path +from b2sdk._internal.sync.action import B2CopyAction, B2DownloadAction +from test.known_defect import known_defect + +pytestmark = [pytest.mark.apiver(from_ver=2)] + +REMOTE_MTIME_MILLIS = 1_000_000_000_000 # 2001-09-09 + + +@pytest.fixture +def api_and_bucket(): + api = B2Api( + InMemoryAccountInfo(), + cache=InMemoryCache(), + api_config=B2HttpApiConfig(_raw_api_class=RawSimulator), + ) + account_id, key = api.session.raw_api.create_account() + api.authorize_account(application_key_id=account_id, application_key=key, realm='production') + return api, api.create_bucket('sync-bucket', 'allPublic') + + +def _download_action(bucket, file_version, local_path): + path = B2Path(file_version.file_name, file_version, [file_version]) + return B2DownloadAction( + path, + file_version.file_name, + str(local_path), + ServerDefaultSyncEncryptionSettingsProvider(), + ) + + +def _corrupt_stored_sha1(api, bucket, file_version): + sim_bucket = api.session.raw_api.bucket_id_to_bucket[bucket.id_] + sim_bucket.file_id_to_file[file_version.id_].content_sha1 = 'f' * 40 + + +class TestDownloadedFileModTime: + @known_defect( + 'C3', 'a download that fails its checksum check still gets the remote modification time' + ) + def test_c3_failed_checksum_does_not_stamp_remote_mtime(self, api_and_bucket, tmp_path): + """C3: a download that fails validation must not leave a file that looks current.""" + api, bucket = api_and_bucket + fv = bucket.upload_bytes( + b'hello world', + 'f.txt', + file_info={'src_last_modified_millis': str(REMOTE_MTIME_MILLIS)}, + ) + _corrupt_stored_sha1(api, bucket, fv) + target = tmp_path / 'out.txt' + with pytest.raises((ChecksumMismatch, TruncatedOutput)): + bucket.download_file_by_id(fv.id_).save_to(target) + # either the file is gone, or it does not carry the remote mod time + if target.exists(): + assert ( + int(target.stat().st_mtime * 1000) != REMOTE_MTIME_MILLIS + ), 'the remote modification time was set on a file that failed validation' + + +class TestSyncDownloadAction: + @known_defect('C3', 'a failed sync download leaves the temporary file behind') + def test_c3_failed_download_leaves_no_temp_file(self, api_and_bucket, tmp_path): + """C3: after a failed sync download the .b2.sync.tmp file must not be left behind.""" + api, bucket = api_and_bucket + fv = bucket.upload_bytes(b'hello world', 'f.txt') + _corrupt_stored_sha1(api, bucket, fv) + action = _download_action(bucket, fv, tmp_path / 'f.txt') + with pytest.raises((ChecksumMismatch, TruncatedOutput)): + action.do_action(bucket, None) + assert not (tmp_path / 'f.txt').exists(), 'final file was created' + assert not (tmp_path / 'f.txt.b2.sync.tmp').exists(), 'temporary file left behind' + + @known_defect( + 'C17', 'sync download deletes the old local file before renaming the new one into place' + ) + def test_c17_failed_rename_keeps_previous_local_copy(self, api_and_bucket, tmp_path): + """C17: if moving the new file into place fails, the old local file must still exist.""" + api, bucket = api_and_bucket + fv = bucket.upload_bytes(b'new content', 'f.txt') + local = tmp_path / 'f.txt' + local.write_bytes(b'old content') + action = _download_action(bucket, fv, local) + with mock.patch('os.rename', side_effect=OSError('simulated rename failure')): + with pytest.raises(OSError): + action.do_action(bucket, None) + assert local.exists(), 'the previous local copy was removed' + assert local.read_bytes() == b'old content', 'the previous local copy was changed' + + def test_c17_guard_successful_download_replaces_file(self, api_and_bucket, tmp_path): + """Guard for C17: a normal sync download replaces the local file and leaves no temp file.""" + api, bucket = api_and_bucket + fv = bucket.upload_bytes(b'new content', 'f.txt') + local = tmp_path / 'f.txt' + local.write_bytes(b'old content') + _download_action(bucket, fv, local).do_action(bucket, None) + assert local.read_bytes() == b'new content', 'the download did not replace the file' + assert not os.path.exists(str(local) + '.b2.sync.tmp'), 'temporary file left behind' + + +class TestSyncMessages: + NAME = 'dir/evil\x1b[31mname' + + def _path(self): + return B2Path(self.NAME, mock.Mock(), []) + + @known_defect( + 'C24', 'the sync download message prints remote names without escaping control characters' + ) + def test_c24_download_message_escapes_control_characters(self): + """C24: remote names with control chars must be escaped in output.""" + action = B2DownloadAction(self._path(), self.NAME, 'out/x', mock.Mock()) + reporter = mock.Mock() + action.do_report(mock.Mock(), reporter) + printed = reporter.print_completion.call_args[0][0] + assert '\x1b' not in printed, repr(printed) + + @known_defect( + 'C24', 'the sync copy message prints remote names without escaping control characters' + ) + def test_c24_copy_message_escapes_control_characters(self): + """C24: same for copy actions.""" + action = B2CopyAction( + self.NAME, self._path(), 'dest', mock.Mock(), mock.Mock(), mock.Mock() + ) + reporter = mock.Mock() + action.do_report(mock.Mock(), reporter) + printed = reporter.print_completion.call_args[0][0] + assert '\x1b' not in printed, repr(printed) diff --git a/test/unit/test_regression_misc.py b/test/unit/test_regression_misc.py new file mode 100644 index 000000000..6098c4241 --- /dev/null +++ b/test/unit/test_regression_misc.py @@ -0,0 +1,124 @@ +###################################################################### +# +# File: test/unit/test_regression_misc.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Regression tests for validation helpers, folder scanning, the HTTP layer and types. + +Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct +behaviour and are strict expected failures, so the default run stays green and fails with +"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run +``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C`` identifiers are +short labels that also appear in the test names and decorators. +""" + +from __future__ import annotations + +import importlib +import os +import sys +from unittest import mock + +import pytest +import requests +import responses +from apiver_deps import B2Http, B2HttpApiConfig + +from b2sdk._internal.scan.folder import LocalFolder +from b2sdk._internal.scan.policies import DEFAULT_SCAN_MANAGER +from test.known_defect import known_defect + +pytestmark = [pytest.mark.apiver(from_ver=2)] + +URL = 'https://example.com/b2api/v4/test' + + +def _b2_http(): + return B2Http(B2HttpApiConfig(requests.Session, install_clock_skew_hook=False)) + + +@pytest.mark.skipif(sys.platform == 'win32', reason='needs symlinks') +class TestLocalFolderSymlinks: + @known_defect('C53', 'the symlink loop check compares only inode numbers, not devices') + def test_c53_symlinks_to_different_devices_with_same_inode_are_not_loops(self, tmp_path): + """C53: links to directories on another device that share an inode number are not loops.""" + t1, t2 = tmp_path / 't1', tmp_path / 't2' + t1.mkdir() + t2.mkdir() + (t1 / 'f1.txt').write_text('1') + (t2 / 'f2.txt').write_text('2') + root = tmp_path / 'root' + root.mkdir() + os.symlink(t1, root / 'l1') + os.symlink( + t2, t1 / 'l2' + ) # nested link: the loop check only tracks links on the current path + + real_stat = type(t1).stat + + def fake_stat(self, *args, **kwargs): + st = real_stat(self, *args, **kwargs) + if self.name == 't2': # pretend t2 lives on another device but has t1's inode number + t1_st = real_stat(t1) + values = list(st) + values[1] = t1_st.st_ino + values[2] = t1_st.st_dev + 1 + return os.stat_result(values) + return st + + with mock.patch.object(type(t1), 'stat', fake_stat): + names = sorted( + p.relative_path + for p in LocalFolder(str(root)).all_files(None, DEFAULT_SCAN_MANAGER) + ) + assert names == ['l1/f1.txt', 'l1/l2/f2.txt'] + + +class TestTypes: + @known_defect( + 'C31', 'the pydantic version is compared as text, so "10.0" counts as older than "2"' + ) + def test_c31_pydantic_version_ten_counts_as_v2_or_newer(self, monkeypatch): + """C31: pydantic version comparison must be numeric ('10.0' is newer than '2').""" + pydantic = pytest.importorskip('pydantic') + types_module = importlib.import_module('b2sdk._internal.types') + monkeypatch.setattr(pydantic, '__version__', '10.0.0') + try: + importlib.reload(types_module) + detected = types_module.pydantic + finally: + monkeypatch.undo() + importlib.reload(types_module) + assert detected is not None, 'pydantic was not detected' + + +class TestHttpLayer: + @responses.activate + @known_defect('C10', 'calls that create things are re-sent after a timeout') + def test_c10_create_style_post_is_not_blindly_resent_after_timeout(self): + """C10: a POST that creates something (b2_create_bucket) must not be re-sent after a timeout.""" + url = 'https://example.com/b2api/v4/b2_create_bucket' + responses.add(responses.POST, url, body=requests.exceptions.ReadTimeout('timed out')) + http = _b2_http() + with mock.patch('time.sleep'): + with pytest.raises(Exception): + http.post_json_return_json(url, {}, {'bucketName': 'b'}) + assert len(responses.calls) == 1, f'request was sent {len(responses.calls)} times' + + @pytest.mark.parametrize('args', [()]) + @known_defect('C51', 'error mapping raises IndexError for a ConnectionError with no arguments') + def test_c51_connection_error_with_missing_args_is_mapped_without_indexerror(self, args): + """C51: ConnectionError carrying fewer than 2 args must map to a B2 error, not IndexError.""" + from b2sdk._internal.b2http import B2Http as InternalB2Http + + error = requests.ConnectionError(*args) + try: + InternalB2Http._translate_errors(lambda: (_ for _ in ()).throw(error)) + except IndexError: + pytest.fail('IndexError raised for ConnectionError with missing args') + except Exception: + pass diff --git a/test/unit/v_all/test_regression_legacy_api.py b/test/unit/v_all/test_regression_legacy_api.py new file mode 100644 index 000000000..e924301cf --- /dev/null +++ b/test/unit/v_all/test_regression_legacy_api.py @@ -0,0 +1,62 @@ +###################################################################### +# +# File: test/unit/v_all/test_regression_legacy_api.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +"""Regression tests for the legacy v0/v1 interfaces. + +Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct +behaviour and are strict expected failures, so the default run stays green and fails with +"XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run +``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C`` identifiers are +short labels that also appear in the test names and decorators. +""" + +from __future__ import annotations + +import apiver_deps +import pytest +from apiver_deps import B2Api, RawSimulator, StubAccountInfo + +from test.known_defect import known_defect + +if apiver_deps.V <= 1: + from apiver_deps import DownloadDestBytes +else: + DownloadDestBytes = None # not present in v2+; tests are skipped there + +pytestmark = [pytest.mark.apiver(to_ver=1)] + + +@pytest.fixture +def bucket(): + simulator = RawSimulator() + api = B2Api(StubAccountInfo(), raw_api=simulator) + account_id, master_key = simulator.create_account() + api.authorize_account('production', account_id, master_key) + return api.create_bucket('my-bucket', 'allPublic') + + +@pytest.mark.apiver(0) +@known_defect('C48', 'v0 unhide_file() raises TypeError') +def test_c48_v0_unhide_file_removes_hide_marker(bucket): + """C48: on the v0 interface unhide_file() must not fail with a TypeError.""" + bucket.upload_bytes(b'hello', 'hello.txt') + bucket.hide_file('hello.txt') + bucket.unhide_file('hello.txt') + names = [info.file_name for info, _ in bucket.ls(show_versions=False)] + assert 'hello.txt' in names, names + + +@known_defect('C44', 'v0/v1 clients cannot download a file stored with a Content-Encoding') +def test_c44_legacy_api_can_download_file_stored_with_content_encoding(bucket): + """C44: v0/v1 clients must be able to download a file that has a Content-Encoding.""" + data = b'compressed-looking payload' + bucket.upload_bytes(data, 'enc.bin', file_infos={'b2-content-encoding': 'gzip'}) + dest = DownloadDestBytes() + bucket.download_file_by_name('enc.bin', dest) + assert dest.get_bytes_written() == data, 'wrong downloaded data'