Repository navigation
Add regression tests for known defects #673
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Add regression tests for known defects; tests of defects that are still present are strict expected failures marked `known_defect` (run them with `pytest --runxfail -m known_defect`). |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| ###################################################################### | ||
| # | ||
| # File: test/known_defect.py | ||
| # | ||
| # Copyright 2026 Backblaze Inc. All Rights Reserved. | ||
| # | ||
| # License https://www.backblaze.com/using_b2_code.html | ||
| # | ||
| ###################################################################### | ||
| from __future__ import annotations | ||
|
|
||
| import pytest | ||
|
|
||
|
|
||
| def known_defect(defect_id: str, summary: str, *, condition: bool = True): | ||
| """ | ||
| Mark a test that asserts the correct behaviour of a defect that is still present. | ||
|
|
||
| The test is a strict expected failure (``xfail(strict=True)``), so the default test run stays | ||
| green, and the run fails as soon as the defect is fixed ("XPASS(strict)"). When that happens, | ||
| delete the ``@known_defect(...)`` line: the test then becomes a normal regression test. | ||
|
|
||
| The tests carry the ``known_defect`` marker, so they can be run (and are reported as real | ||
| failures) on demand with one command, for example:: | ||
|
|
||
| pytest --runxfail -m known_defect --api=v3 test/unit test/static | ||
|
|
||
| :param defect_id: identifier of the defect, for example ``'C16'`` | ||
| :param summary: one-line description of the defect | ||
| :param condition: when false the defect does not apply (for example to another apiver) and the | ||
| test is a normal test | ||
| """ | ||
|
|
||
| def decorator(func): | ||
| if not condition: | ||
| return func | ||
| func = pytest.mark.known_defect(func) | ||
| return pytest.mark.xfail(strict=True, reason=f'Known defect {defect_id}: {summary}')(func) | ||
|
|
||
| return decorator |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| ###################################################################### | ||
| # | ||
| # File: test/static/test_regression_repo_checks.py | ||
| # | ||
| # Copyright 2026 Backblaze Inc. All Rights Reserved. | ||
| # | ||
| # License https://www.backblaze.com/using_b2_code.html | ||
| # | ||
| ###################################################################### | ||
| """Static regression checks of repository files (docs, changelog, packaging, sources). | ||
|
|
||
| They read repository files only and are skipped when the repository files are not available. | ||
|
|
||
| Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct | ||
| behaviour and are strict expected failures, so the default run stays green and fails with | ||
| "XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run | ||
| ``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C<n>`` identifiers are | ||
| short labels that also appear in the test names and decorators. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import re | ||
| import subprocess | ||
| import sys | ||
| import textwrap | ||
| from pathlib import Path | ||
|
|
||
| import pytest | ||
|
|
||
| from test.known_defect import known_defect | ||
|
|
||
| ROOT = Path(__file__).resolve().parents[2] | ||
|
|
||
|
|
||
| def _read(rel: str) -> str: | ||
| path = ROOT / rel | ||
| if not path.is_file(): | ||
| pytest.skip(f'{rel} is not available (not a full repository checkout)') | ||
| return path.read_text(encoding='utf-8') | ||
|
|
||
|
|
||
| @known_defect('C30', 'typo in .github/SUPPORT.md') | ||
| def test_c30_support_file_has_no_known_typo(): | ||
| """C30: typo 'Issuew' in .github/SUPPORT.md (codespell did not catch it).""" | ||
| assert 'Issuew' not in _read('.github/SUPPORT.md'), 'typo is still present' | ||
|
|
||
|
|
||
| @known_defect('C33', 'Bucket.upload leaves a FIXME about min_part_size semantics') | ||
| def test_c33_min_part_size_fixme_resolved(): | ||
| """C33: min_part_size docs/code mismatch must not be left as a FIXME in Bucket.upload.""" | ||
| assert 'FIXME: Bucket.upload documents wrong logic' not in _read( | ||
| 'b2sdk/_internal/bucket.py' | ||
| ), 'the FIXME is still present' | ||
|
|
||
|
|
||
| @known_defect('C59', 'the README does not mention the "full" install extra') | ||
| def test_c59_readme_mentions_full_extra(): | ||
| """C59: README must mention the `full` install extra (tqdm/pydantic support).""" | ||
| assert re.search(r'b2sdk\[full\]', _read('README.md')), 'README does not mention b2sdk[full]' | ||
|
|
||
|
|
||
| @known_defect('C4', 'the Quick Start uses a class and an argument name that do not exist') | ||
| def test_c4_quick_start_uses_existing_names(): | ||
| """C4: Quick Start must not use the non-existent class EncryptionSettings or the wrong `id=` argument.""" | ||
| text = _read('doc/source/quick_start.rst') | ||
| assert 'EncryptionSettings(' not in text, 'EncryptionSettings does not exist' | ||
| assert not re.search(r'EncryptionKey\([^)]*\bid=', text), 'EncryptionKey has no id= argument' | ||
|
|
||
|
|
||
| @known_defect( | ||
| 'C4', 'the development-version install instructions install from PyPI, not the checkout' | ||
| ) | ||
| def test_c4_install_doc_development_version_installs_from_checkout(): | ||
| """C4: 'Installing a development version' must install the checkout, not PyPI.""" | ||
| section = _read('doc/source/install.rst').split('Installing a development version', 1)[1] | ||
| assert re.search( | ||
| r'pip install (-e )?[.\w/]*\.(\s|$)|pip install -e', section | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Require a checkout target after The second alternative accepts Require a local checkout target for both alternatives. Preserve valid commands such as
|
||
| ), 'development-version section installs from PyPI' | ||
|
|
||
|
|
||
| @known_defect('C36', 'importing b2sdk fails when the package metadata is not installed') | ||
| def test_c36_version_import_works_when_package_metadata_is_missing(tmp_path): | ||
| """C36: importing a vendored/uninstalled copy of b2sdk must not fail for missing dist metadata.""" | ||
| script = textwrap.dedent( | ||
| f""" | ||
| import importlib.metadata as m, sys | ||
| orig = m.version | ||
| def fake(name): | ||
| if name == 'b2sdk': | ||
| raise m.PackageNotFoundError(name) | ||
| return orig(name) | ||
| m.version = fake | ||
| sys.path.insert(0, {str(ROOT)!r}) | ||
| import b2sdk.version | ||
| """ | ||
| ) | ||
| result = subprocess.run( | ||
| [sys.executable, '-c', script], capture_output=True, text=True, cwd=tmp_path | ||
| ) | ||
| assert result.returncode == 0, result.stderr[-300:] | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| ###################################################################### | ||
| # | ||
| # File: test/unit/account_info/test_regression_sqlite.py | ||
| # | ||
| # Copyright 2026 Backblaze Inc. All Rights Reserved. | ||
| # | ||
| # License https://www.backblaze.com/using_b2_code.html | ||
| # | ||
| ###################################################################### | ||
| """Regression tests for SqliteAccountInfo. | ||
|
|
||
| Tests marked with ``@known_defect`` describe a defect that is still present: they assert the correct | ||
| behaviour and are strict expected failures, so the default run stays green and fails with | ||
| "XPASS(strict)" once the defect is fixed (then remove the decorator). To see them fail, run | ||
| ``pytest --runxfail -m known_defect`` (see ``test/known_defect.py``). The ``C<n>`` identifiers are | ||
| short labels that also appear in the test names and decorators. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import os | ||
| import threading | ||
|
|
||
| import pytest | ||
| from apiver_deps import ( | ||
| B2_ACCOUNT_INFO_ENV_VAR, | ||
| XDG_CONFIG_HOME_ENV_VAR, | ||
| SqliteAccountInfo, | ||
| ) | ||
|
|
||
| from test.known_defect import known_defect | ||
|
|
||
| pytestmark = [pytest.mark.apiver(from_ver=2)] | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def home(monkeypatch, tmp_path): | ||
| monkeypatch.setenv('HOME', str(tmp_path)) | ||
| monkeypatch.setenv('USERPROFILE', str(tmp_path)) | ||
| monkeypatch.delenv(B2_ACCOUNT_INFO_ENV_VAR, raising=False) | ||
| monkeypatch.delenv(XDG_CONFIG_HOME_ENV_VAR, raising=False) | ||
| return tmp_path | ||
|
|
||
|
|
||
| class TestProfilePaths: | ||
| @known_defect( | ||
| 'C47', | ||
| 'an existing ~/.b2db-<profile>.sqlite is ignored because "~" is not expanded before the existence check', | ||
| ) | ||
| def test_c47_existing_legacy_profile_database_is_used_with_xdg(self, home, monkeypatch): | ||
| """C47: an existing ~/.b2db-<profile>.sqlite must be found even when XDG_CONFIG_HOME is set.""" | ||
| monkeypatch.setenv(XDG_CONFIG_HOME_ENV_VAR, str(home / 'xdg')) | ||
| legacy = home / '.b2db-foo.sqlite' | ||
| legacy.write_bytes(b'') | ||
| path = SqliteAccountInfo.get_user_account_info_path(profile='foo') | ||
| assert os.path.realpath(path) == os.path.realpath(str(legacy)), f'got {path}' | ||
|
|
||
|
|
||
| class TestThreadConnections: | ||
| @known_defect('C37', "close() closes only the calling thread's database connection") | ||
| def test_c37_close_closes_connections_of_other_threads(self, tmp_path): | ||
| """C37: close() must close the database connections opened by other threads, too.""" | ||
| info = SqliteAccountInfo(file_name=str(tmp_path / 'acct.sqlite')) | ||
| opened, closed_by_main, result = threading.Event(), threading.Event(), {} | ||
|
|
||
| def worker(): | ||
| conn = info._get_connection() # connection of this worker thread | ||
| opened.set() | ||
| assert closed_by_main.wait(timeout=10), 'main thread did not call close()' | ||
| try: | ||
| conn.execute('select 1') | ||
| result['closed'] = False | ||
| except Exception: # sqlite3.ProgrammingError: Cannot operate on a closed database | ||
| result['closed'] = True | ||
|
|
||
| thread = threading.Thread(target=worker) | ||
| thread.start() | ||
| assert opened.wait(timeout=10), 'worker thread did not open a connection' | ||
| info.close() | ||
| closed_by_main.set() | ||
| thread.join(timeout=10) | ||
| assert ( | ||
| result.get('closed') is True | ||
| ), 'connection of another thread is still open after close()' |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Detect
id=after parentheses inside the secret.The existing Quick Start secret contains
)beforeid=. The pattern's[^)]*stops inside that string, so it misses the invalid argument. If onlyEncryptionSettingsis corrected, this test passes while theid=defect remains. (raw.githubusercontent.com)Use a quote-aware argument check. Verify that the existing secret fails with
id=and passes withkey_id=.