Release 1.9.0

.github/release_checklist.md

@@ -1,15 +1,15 @@
 Release checklist
-- [ ] Check outstanding issues on JIRA and Github.
+- [ ] Check outstanding issues on GitHub.
 - [ ] Check [latest documentation](https://python-isal.readthedocs.io/en/latest/) looks fine.
 - [ ] Create a release branch.
   - [ ] Change current development version in `CHANGELOG.rst` to stable version.
 - [ ] Check if the address sanitizer does not find any problems using `tox -e asan`
 - [ ] Merge the release branch into `main`.
 - [ ] Created an annotated tag with the stable version number. Include changes 
 from CHANGELOG.rst.
-- [ ] Push tag to remote. This triggers the wheel/sdist build on github CI.
+- [ ] Push tag to remote. This triggers the wheel/sdist build on GitHub CI.
 - [ ] merge `main` branch back into `develop`.
 - [ ] Build the new tag on readthedocs. Only build the last patch version of
 each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`.
-- [ ] Create a new release on github.
+- [ ] Create a new release on GitHub.
 - [ ] Update the package on conda-forge.

.github/workflows/ci.yml

@@ -16,10 +16,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install tox
         run: pip install tox
       - name: Lint
@@ -31,15 +31,17 @@ jobs:
         tox_env:
           - docs
           - twine_check
-    runs-on: ubuntu-latest
+    # Use the same Ubuntu release as build.os in .readthedocs.yml, so the
+    # documentation is checked against the libisal that Read the Docs uses.
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install isal
         run: sudo apt-get install libisal-dev
       - name: Install tox and upgrade setuptools and pip
@@ -54,28 +56,29 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.10"
           - "3.11"
           - "3.12"
           - "3.13"
           - "3.14"
           - "3.14t"
-          - "pypy-3.10"
+          - "3.15"
+          - "3.15t"
           - "pypy-3.11"
+          - "pypy-3.12"
         os: ["ubuntu-latest"]
         include:
-          - os: "macos-13"
-            python-version: "3.10"
+          - os: "macos-15-intel"
+            python-version: "3.11"
           - os: "macos-latest"
-            python-version: "3.10"
+            python-version: "3.11"
           - os: "windows-latest"
-            python-version: "3.10"
+            python-version: "3.11"
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
@@ -106,7 +109,7 @@ jobs:
     strategy:
       matrix:
         python_version:
-          - "3.10"
+          - "3.11"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -136,17 +139,14 @@ jobs:
         shell: bash -l {0}
     strategy:
       matrix:
-        os: ["ubuntu-latest", "macos-13", "windows-latest"]
-        python_version: [ "python" ]
-        include:
-          - os: "ubuntu-latest"
-            python_version: "pypy"
+        os: ["ubuntu-latest", "macos-15-intel", "windows-latest"]
+        python_version: [ "python==3.11" ]
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Install miniconda.
-        uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda.
+        uses: conda-incubator/setup-miniconda@v4 # https://github.com/conda-incubator/setup-miniconda.
         with:
           channels: conda-forge,defaults
       - name: Install requirements (universal)
@@ -159,6 +159,49 @@ jobs:
         env:
           PYTHON_ISAL_LINK_DYNAMIC: True
 
+  # Run the tests on CPython builds with AddressSanitizer and ThreadSanitizer
+  # from https://github.com/nascheme/cpython_sanity.
+  test-sanitizers:
+    runs-on: ubuntu-latest
+    # A crash under ThreadSanitizer can hang instead of exiting.
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sanitizer: address
+            image: ghcr.io/nascheme/cpython-asan:3.14
+          # The free-threaded build runs Python threads in parallel.
+          - sanitizer: thread
+            image: ghcr.io/nascheme/cpython-tsan:3.14t
+    container:
+      image: ${{ matrix.image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install build dependencies
+        run: apt-get update && apt-get install -y nasm
+      - name: Install python-isal with ${{ matrix.sanitizer }} sanitizer
+        # setuptools-scm needs git, and the container user does not own the
+        # repository.
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          python -m pip install . pytest pytest-timeout
+        env:
+          # The extension modules get -fsanitize from CPython. CFLAGS adds it
+          # to the C code of ISA-L.
+          CFLAGS: -fsanitize=${{ matrix.sanitizer }} -fno-omit-frame-pointer -g
+      - name: Run tests
+        # Do not set PYTHONDEVMODE. Its debug allocator pads each allocation,
+        # so AddressSanitizer misses small overflows. --capture=sys prints
+        # sanitizer reports from C code even when they end the process, and
+        # -v prints the name of the test before its report.
+        run: python -m pytest -v tests --capture=sys
+        env:
+          # Exit at the first data race. A crash after a race can hang.
+          TSAN_OPTIONS: halt_on_error=1
+
   deploy:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
     runs-on: ${{ matrix.os }}
@@ -168,12 +211,13 @@ jobs:
       - test-static
       - test-dynamic
       - test-arch
+      - test-sanitizers
     strategy:
       matrix:
         os:
           - ubuntu-latest
-          - macos-13
-          - macos-14
+          - macos-15-intel
+          - macos-latest
           - windows-latest
         cibw_archs_linux: ["x86_64"]
         cibw_before_all_linux:
@@ -195,7 +239,7 @@ jobs:
         with:
           submodules: recursive
           fetch-depth: 0  # Fetch everything to get accurately versioned tag.
-      - uses: actions/setup-python@v2  # Some issues where caused by higher versions.
+      - uses: actions/setup-python@v7
         name: Install Python
       - name: Install cibuildwheel twine wheel
         run: python -m pip install cibuildwheel twine wheel
@@ -216,6 +260,7 @@ jobs:
       - name: Build wheels
         run: cibuildwheel --output-dir dist
         env:
+          CIBW_ENABLE: "pypy"
           CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64"  # Skip 32 bit and problematic mac builds.
           CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }}
           CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }}

CHANGELOG.rst

@@ -8,6 +8,151 @@ Changelog
 .. that users understand how the changes affect the new version.
 
 
+version 1.9.0
+-----------------
+
+By far most of the work on this release has been done by
+`@ptweezy <https://github.com/ptweezy>`_. Almost all bug fixes and improvements
+are their work.
+
+Major changes:
+
++ Updated included ISA-L library to 2.32.1.
++ Python 3.10 and PyPy 3.10 are no longer supported. Added support for PyPy
+  3.12 and Python 3.15.
++ Restore PyPy wheel builds.
++ Fix builds on macOS with Apple Silicon, which used the portable C code of
+  ISA-L rather than its optimized aarch64 assembly. On an Apple M4,
+  decompression is 1.6 to 3 times faster, CRC-32 is 50 times faster, and
+  Adler-32 is 7 times faster.
+
+Optimizations and features:
+
++ ``isal_zlib.compressobj`` supports ``strategy=Z_FIXED`` at compression
+  level 0. Level 0 then uses fixed Huffman codes instead of starting each
+  block with a Huffman table of about 110 bytes. This shrinks small flushed
+  writes, such as websocket messages: JSON messages of about 440 bytes, each
+  followed by ``flush(Z_SYNC_FLUSH)``, compress to an average of 75 instead
+  of 188 bytes. Blocks of a few KiB or more can be larger: about 5% for JSON
+  and 10% for Python source code at 64 KiB. Other strategies, and ``Z_FIXED``
+  at levels 1 to 3, still give a warning and use the default strategy.
++ Speed up ``import isal.igzip`` and ``import isal.igzip_threaded`` by
+  importing ``argparse`` and ``shutil`` only when the command line interface
+  runs, and by dropping the ``multiprocessing`` import.
++ Speed up reading lines from binary ``igzip`` files through iteration,
+  ``readlines()``, and ``readline()``. On an Apple M4, reading FASTQ data is
+  1.13 times faster, and reading lines of a few bytes is 1.5 times faster.
++ Speed up each ``write()`` call to binary ``igzip`` files on CPython. On an
+  AMD Ryzen 7 9800X3D with Python 3.14.8, writing FASTQ data or Python source
+  code one line at a time is 1.1 times faster.
++ Speed up many small writes to ``igzip.IGzipFile`` on Python 3.12 and later.
+  ``IGzipFile`` collects these writes in the write buffer of
+  ``gzip.GzipFile`` and compresses them in large chunks, which makes writing
+  lines 1.4 times faster.
+
+Bug fixes:
+
++ Fix zlib streams compressed with a preset dictionary so they can be decoded
+  by standard zlib, including builds linked against older ISA-L releases.
++ Fix backward seeking in igzip files by discarding buffered compressed data
+  when rewinding the underlying file.
++ Fix a crash when ``igzip.decompress`` gets an object that is not
+  bytes-like, such as a ``str``. It raises a ``TypeError`` instead.
++ Fix a crash after creating ``igzip_lib.IgzipDecompressor`` objects with a
+  ``zdict``.
++ Fix error handling when more than one interpreter imports
+  ``isal.isal_zlib`` or ``isal.igzip_lib``, for example, the main interpreter
+  and a legacy subinterpreter that ``Py_NewInterpreter()`` creates. Each
+  import replaced the error classes that the C code raises, so
+  ``except isal_zlib.error``, ``except igzip_lib.error``, and
+  ``except gzip.BadGzipFile`` caught errors only in the interpreter that
+  imported the modules last.
++ Fix a hang in ``isal_zlib.Compress``: after ``flush()`` raised an error for
+  an unsupported mode, the next call on the same object never returned.
++ Fix a crash when you call ``flush(Z_FINISH)`` twice on a level 0
+  ``isal_zlib.Compress`` object.
++ Fix ``unused_data`` of ``isal_zlib.Decompress`` and
+  ``igzip_lib.IgzipDecompressor``, which could contain bytes from outside the
+  input buffer when ``max_length`` split the end of a stream. At the end of a
+  stream, ``isal_zlib.Decompress.unconsumed_tail`` is empty, matching the
+  CPython fix for gh-158169.
++ Fix ``igzip_lib.IgzipDecompressor.needs_input``, which could be ``True``
+  after ``decompress()`` returned ``max_length`` bytes while more output was
+  still available. Readers that stop at the end of the file when
+  ``needs_input`` is ``True``, such as CPython's ``DecompressReader``, raised
+  ``EOFError`` for valid data.
++ Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47, which detects
+  zlib or gzip data. It failed on gzip data when the first ``decompress()``
+  call got fewer than 2 bytes, for example, when the data arrived byte by
+  byte.
++ Fix ``isal_zlib.decompressobj`` with a ``wbits`` of 40 to 47 after a
+  ``flush()`` call that came before any data. It then failed on both zlib and
+  gzip data.
++ ``igzip_threaded.open`` raises ``ValueError`` for a ``block_size`` of about
+  3.6 GiB or more and ``TypeError`` for a ``compresslevel`` that is not an
+  integer. Both raised ``SystemError`` before.
++ Fix ``igzip_threaded`` writers at compression level 0, which raised
+  ``OverflowError`` for data that does not compress, such as random bytes or
+  already compressed files. At compression level 0, ``block_size`` must now be
+  less than about 2 GiB.
++ Extraordinarily large gzip headers with either a very long comment or
+  filename will now cause an ``OverflowError`` rather than use more and more
+  memory. The size is limited to the internal buffer size which is currently
+  set at 512 KiB.
++ Fix ``igzip_threaded`` readers and writers that waited for a polling timeout
+  of up to 50 ms at the end of the stream or when closed. Writing and closing
+  a small file takes less than a millisecond instead of 60 ms.
++ Fix ``igzip_threaded`` writers that waited forever when writing to the
+  underlying file failed, for example, because the disk was full. A later call
+  to ``write()``, ``flush()``, or ``close()`` raises the error. After an error,
+  ``close()`` no longer closes a file object that you passed to
+  ``igzip_threaded.open``.
++ Fix ``igzip_threaded`` writers with more than one thread that waited
+  forever when compressing a block failed, for example, because there was not
+  enough memory. A later call to ``write()``, ``flush()``, or ``close()``
+  raises the error.
++ Fix a crash on the free-threaded build when one thread reads
+  ``unused_data`` or ``unconsumed_tail`` of an ``isal_zlib`` decompressor
+  while another thread decompresses with it. Reading the attributes of
+  ``isal_zlib`` decompressors and ``igzip_lib.IgzipDecompressor`` from other
+  threads is now safe, and so is calling ``decompress()`` from several
+  threads at the same time when ``wbits`` detects the format.
++ ``isal_zlib.crc32_combine`` and the ``read()`` and ``seek()`` methods of the
+  private ``isal_zlib._GzipReader`` class raise ``TypeError`` instead of
+  ``SystemError`` for arguments of the wrong type or number.
+  ``igzip.IGzipFile`` was not affected.
++ Fix a crash on PyPy when you create an ``isal_zlib.Compress`` or
+  ``isal_zlib.Decompress`` object by calling the type, for example,
+  ``isal_zlib.Compress()``, and then use it. Calling the type raises
+  ``TypeError`` now, as it does on CPython. Use ``compressobj()`` and
+  ``decompressobj()`` instead.
++ Fix ``readinto()`` of the private ``isal_zlib._GzipReader`` class, which
+  wrote into read-only objects such as ``bytes``. It raises ``TypeError`` now.
+  ``igzip.IGzipFile`` was not affected.
++ Fix the ``mtime`` attribute of ``igzip.IGzipFile`` objects in read mode,
+  which was ``None`` for gzip files with a modification time of 0. It is 0 now,
+  as in the ``gzip`` module. ``gzip -n``, ``python -m isal.igzip -n``, and
+  ``igzip_threaded`` writers make such files, and so does ``gzip.compress`` by
+  default on Python 3.14.
++ Fix a crash when there is not enough memory to create an
+  ``igzip_lib.IgzipDecompressor``. It raises ``MemoryError`` now.
++ Fix a crash when there is not enough memory to raise ``BadGzipFile`` for
+  data that is not in the gzip format, for example, in ``igzip.decompress``.
+  It raises ``MemoryError`` now.
++ Fix a crash in ``igzip_lib.IgzipDecompressor.decompress()`` after a call on
+  the same object raised ``MemoryError``. The next call used the input of the
+  failed call after the caller had released it. This matches the CPython fix
+  for gh-148395.
++ Fix a crash on macOS and PyPy when there is not enough memory to create the
+  internal lock of an ``isal_zlib.Compress``, ``isal_zlib.Decompress``, or
+  ``igzip_lib.IgzipDecompressor`` object. It raises ``MemoryError`` now.
++ Fix the ``-0`` and ``--fast`` options of ``python -m isal.igzip``, which
+  compressed at level 2 instead of level 0.
++ Fix crashes and wrong data when one thread calls ``seek()`` of the
+  private ``isal_zlib._GzipReader`` class while another thread reads from the
+  same object. ``igzip.IGzipFile`` was not affected.
+
+
 version 1.8.0
 -----------------
 + Python 3.14 is supported.

README.rst

@@ -113,12 +113,12 @@ your project please list a python-isal dependency as follows.
 ``setup.cfg``::
 
     install_requires =
-        isal; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64"
+        isal; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64" or platform.machine == "arm64"
 
 ``setup.py``::
 
     extras_require={
-        ":platform.machine == 'x86_64' or platform.machine == 'AMD64' or platform.machine == 'aarch64'": ['isal']
+        ":platform.machine == 'x86_64' or platform.machine == 'AMD64' or platform.machine == 'aarch64' or platform.machine == 'arm64'": ['isal']
     },
 
 .. dependency end
@@ -139,8 +139,9 @@ Differences with zlib and gzip modules
   ``FINISH_FLUSH``. Other flush modes are not supported and will raise errors.
 + ``zlib.Z_DEFAULT_STRATEGY``, ``zlib.Z_RLE`` etc. are exposed as
   ``isal_zlib.Z_DEFAULT_STRATEGY``, ``isal_zlib.Z_RLE`` etc. for compatibility
-  reasons. However, ``isal_zlib`` only supports a default strategy and will
-  give warnings when other strategies are used.
+  reasons. However, ``isal_zlib`` supports only the default strategy, and
+  ``Z_FIXED`` at compression level 0. It gives a warning and uses the default
+  strategy for other strategies, and for ``Z_FIXED`` at levels 1 to 3.
 + ``zlib`` supports different memory levels from 1 to 9 (with 8 default).
   ``isal_zlib`` supports memory levels smallest, small, medium, large and
   largest. These have been mapped to levels 1, 2-3, 4-6, 7-8 and 9. So
@@ -223,6 +224,8 @@ This project builds upon the software and experience of many.  Many thanks to:
 + The `github actions team <https://github.com/orgs/actions/people>`_ for
   creating the actions CI service that enables building and testing on all
   three major operating systems.
++ `@ptweezy <https://github.com/ptweezy>`_ for discovering and fixing lots and
+  lots of bugs as well as suggesting and implementing quite a few improvements.
 + `@animalize <https://github.com/animalize>`_ for explaining how to test and
   build python-isal for ARM 64-bit platforms.
 + And last but not least: everyone who submitted a bug report or a feature