Reconcile main with dev: adata-cli 0.5.0

.github/workflows/publish.yml

@@ -0,0 +1,93 @@
+name: Publish
+
+# Tags are bare MAJOR.MINOR.PATCH, matching the existing convention.
+on:
+  push:
+    tags: ["*"]
+  workflow_dispatch:
+    inputs:
+      target:
+        description: Where to publish
+        required: true
+        default: testpypi
+        type: choice
+        options: [testpypi, pypi]
+
+jobs:
+  # The tag must match the version in pyproject.toml. Nothing enforced this
+  # before, and 0.3.2 was tagged against a tree declaring 0.3.1.
+  check-version:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Tag must match the declared version
+        run: |
+          declared=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2)
+          tag="${GITHUB_REF_NAME}"
+          echo "pyproject: $declared, tag: $tag"
+          if [ "$declared" != "$tag" ]; then
+            echo "::error::Tag $tag does not match pyproject version $declared"
+            exit 1
+          fi
+
+  test:
+    uses: ./.github/workflows/tests.yml
+
+  build:
+    needs: [test]
+    if: always() && needs.test.result == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - run: uv build
+      - name: Check metadata renders on PyPI
+        run: uvx twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-testpypi:
+    needs: [build]
+    if: |
+      always()
+      && needs.build.result == 'success'
+      && github.event_name == 'workflow_dispatch'
+      && inputs.target == 'testpypi'
+    runs-on: ubuntu-latest
+    environment: testpypi
+    permissions:
+      id-token: write   # trusted publishing; no API token secret needed
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  publish-pypi:
+    needs: [build, check-version]
+    # check-version only runs for tags, and GitHub skips a job whose
+    # dependency was skipped -- so without always() a manual dispatch to PyPI
+    # would never publish. Success of build is therefore asserted explicitly,
+    # and check-version is accepted as either passed or not applicable.
+    if: |
+      always()
+      && needs.build.result == 'success'
+      && (needs.check-version.result == 'success' || needs.check-version.result == 'skipped')
+      && (startsWith(github.ref, 'refs/tags/')
+          || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi'))
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/quay-on-tag.yml

@@ -31,7 +31,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: quay.io/cellgeni/h5ad-cli
+          images: quay.io/cellgeni/adata-cli
           tags: |
             type=ref,event=tag
             # push "latest" only for tags WITHOUT hyphens (excludes pre-releases like 1.0.0-beta):

.github/workflows/tests.yml

@@ -3,8 +3,11 @@ name: Tests
 on:
   push:
     branches: [main, dev]
+  # Every pull request, not only those targeting main/dev -- a stacked PR
+  # based on another feature branch would otherwise run no checks at all.
   pull_request:
-    branches: [main, dev]
+  # Reused by publish.yml, so a release cannot skip the suite.
+  workflow_call:
 
 concurrency:
   group: tests-${{ github.workflow }}-${{ github.ref }}
@@ -17,27 +20,15 @@ jobs:
 
     permissions:
       contents: read
-      checks: write   # needed for EnricoMi/publish-unit-test-result-action on PRs
+      checks: write          # publish-unit-test-result-action writes a check run
+      pull-requests: write   # ...and comments the summary on the PR
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.12"]  # add "3.13" if you want
-        module:
-          - name: cli
-            tests: tests/test_cli.py
-          - name: export
-            tests: tests/test_export.py
-          - name: import
-            tests: tests/test_import.py
-          - name: info-read
-            tests: tests/test_info_read.py
-          - name: subset
-            tests: tests/test_subset.py
-          - name: zarr
-            tests: tests/test_zarr.py
-
-    name: tests (${{ matrix.module.name }})
+        python-version: ["3.12", "3.13"]
+
+    name: tests (py${{ matrix.python-version }})
 
     steps:
       - uses: actions/checkout@v4
@@ -57,29 +48,30 @@ jobs:
 
       - name: Run tests with coverage
         run: |
-          uv run pytest -v -W default ${{ matrix.module.tests }} \
-            --cov=h5ad \
+          uv run pytest -v -W default tests/ -m "not integration" \
+            --cov=adata \
             --cov-report=term-missing \
             --cov-report=xml \
             --cov-report=html \
-            --junitxml=pytest-results-${{ matrix.module.name }}.xml
+            --cov-fail-under=90 \
+            --junitxml=pytest-results-py${{ matrix.python-version }}.xml
 
       - name: Publish test results summary
         uses: EnricoMi/publish-unit-test-result-action@v2
         if: always()
         with:
-          files: pytest-results-${{ matrix.module.name }}.xml
-          check_name: Test Results (${{ matrix.module.name }})
+          files: pytest-results-py${{ matrix.python-version }}.xml
+          check_name: Test Results (py${{ matrix.python-version }})
 
       - name: Upload coverage artifacts
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: coverage-${{ matrix.module.name }}
+          name: coverage-py${{ matrix.python-version }}
           path: |
             coverage.xml
             htmlcov/
-            pytest-results-${{ matrix.module.name }}.xml
+            pytest-results-py${{ matrix.python-version }}.xml
           retention-days: 30
 
       - name: Upload coverage to Codecov
@@ -88,3 +80,50 @@ jobs:
         with:
           files: coverage.xml
           fail_ci_if_error: false
+
+  # Builds a store with each pinned anndata release and checks the CLI against
+  # it. Split out because it assembles six environments with uv, which needs
+  # the network and is far slower than the unit suite.
+  compatibility:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    name: anndata 0.8-0.13 compatibility
+
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+
+      - name: Install dependencies (frozen)
+        run: uv sync --extra dev --frozen
+
+      - name: Run compatibility tests
+        env:
+          # A store that will not build must fail this job, not skip it --
+          # otherwise a broken pin leaves it green having checked nothing.
+          ADATA_REQUIRE_VERSION_FIXTURES: "1"
+        run: |
+          uv run pytest -v -W default tests/test_anndata_versions.py \
+            -m integration --junitxml=compat-results.xml
+
+      - name: Confirm the compatibility cases actually ran
+        if: always()
+        run: |
+          # Belt and braces: a marker typo or a collection mistake would make
+          # the step above pass while executing nothing at all.
+          python - <<'EOF'
+          import sys, xml.etree.ElementTree as ET
+          root = ET.parse("compat-results.xml").getroot()
+          suite = root.find("testsuite") if root.tag == "testsuites" else root
+          total = int(suite.get("tests", 0))
+          skipped = int(suite.get("skipped", 0))
+          ran = total - skipped
+          print(f"{ran} compatibility cases ran ({skipped} skipped of {total})")
+          if ran < 150:
+              sys.exit(f"expected at least 150 cases to run, got {ran}")
+          EOF

.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+.venv/
+build/
+dist/
+*.egg-info/
+.coverage
+coverage.xml
+htmlcov/
+.pytest_cache/
+pytest-results*.xml
+compat-results.xml

CHANGELOG.md

@@ -0,0 +1,85 @@
+# Changelog
+
+Notable changes to `adata-cli`. Versions are `MAJOR.MINOR.PATCH`; tags carry no
+`v` prefix.
+
+## 0.5.0
+
+Renamed from `h5ad` to `adata-cli`, restored compatibility with current
+AnnData files, and added four commands.
+
+### Renamed
+
+- Distribution `pyadata-cli` on PyPI (the short name was already taken by
+  an unrelated project), import package `adata`, command `adata`.
+- `info` is now `view`.
+- The `h5ad` command and the `info` subcommand remain as aliases that warn and
+  then run normally. **Both are removed in 1.0.0.**
+
+### Fixed
+
+- **The CLI could not read any file written by anndata >= 0.11.** Since pandas'
+  `future.infer_string` became the default, anndata writes `obs/_index` as a
+  `nullable-string-array` group rather than a dataset, and `axis_len` required
+  a dataset — so `info`, `export dataframe` and `subset` all failed outright.
+- **HDF5 -> Zarr conversion failed on any store with string columns**, i.e. all
+  of them: an h5py variable-length string dataset reports `dtype == object`,
+  which Zarr rejects. All four backend pairings now convert.
+- **`subset` silently dropped `raw/`.** It is now carried over and matched
+  against its own var axis. Unrecognised top-level keys are copied with a
+  warning rather than dropped.
+- **`subset` corrupted group-valued columns**, copying them whole while
+  narrowing everything else. This was invisible before, because files
+  containing such columns could not be read at all.
+- Categoricals no longer degrade to plain strings on a CSV round-trip.
+- `None` round-trips via anndata's `null` encoding instead of an invented
+  `_is_none` marker attribute.
+- `column-order` is honoured on export; previously columns came out in
+  whatever order the backend enumerated (alphabetical on HDF5).
+- Type detection dispatches on `encoding-type` before falling back to
+  structure. A group merely *containing* a member named `obs_names` is no
+  longer misreported as a dataframe.
+- `subset` prefers the declared `_index` over the `obs_names`/`var_names`
+  convention, which had been backwards.
+- A Zarr v2 store is no longer silently upgraded to v3.
+- Chunk shapes are clamped on the axis-column path, which could raise from
+  h5py when subsetting below a column's chunk size.
+
+### Added
+
+- `adata ls` — tree listing for any HDF5 or Zarr store, with no AnnData
+  assumptions, so `.loom` and plain `.h5` work. `--long`, `--depth`, and `-1`
+  for bare paths that pipe into other tools.
+- `adata create` — write a new, empty store for `import` to fill in.
+- `adata split --by <column>` — one store per distinct value, with a CSV
+  manifest. (#2)
+- `adata concat` — concatenate along obs with `--join inner|outer`,
+  `--label`/`--keys`/`--index-unique` and merge strategies for var and uns.
+  Verified to agree with `anndata.concat`.
+- `adata subset --obs-query` / `--var-query` — a small predicate language
+  (`==`, `!=`, `<`, `<=`, `>`, `>=`, `in`, `not in`, `and`, `or`, `not`,
+  parentheses) evaluated while streaming. No new dependency.
+- `adata import image`, and `import` at any path rather than only `obs`/`var`.
+- `--categorical` / `--no-auto-categorical` on `import dataframe`.
+- `export dataframe` from any dataframe path, not only `obs`/`var`. (#4)
+- `export array` can write to stdout. (#4)
+- `adata --version`.
+- `--zarr-format` on the commands that create stores.
+
+### Changed
+
+- Results go to stdout and status to stderr throughout. `view --tree`
+  previously wrote its tree to stderr and only the header to stdout. (#4)
+- Everything written is tagged with its `encoding-type` and
+  `encoding-version`; text is always variable-length UTF-8, as the spec
+  requires, rather than fixed-width bytes.
+- Sparse subsetting streams in blocks instead of loading `data`/`indices`/
+  `indptr` whole.
+- The Docker image ships `duckdb` in place of `csvkit`.
+- CI runs the whole suite on Python 3.12 and 3.13 rather than naming test
+  files individually — which is why `test_storage_root_attrs.py` had never
+  run.
+
+## 0.3.2 and earlier
+
+See the git history. These releases were published to Quay only.

Dockerfile

@@ -1,35 +1,40 @@
 # Base image: Python 3.12 + uv preinstalled (Debian slim)
 FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
 
-# Environment variables
 ENV UV_NO_DEV=1
-ENV VENV=/env
 
-# Work directory inside the container
 WORKDIR /cli
 
-# Install git so we can clone the repo
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends git \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Copy the project files (from GitHub Actions checkout context)
+# Copy the project files (from the GitHub Actions checkout context)
 COPY . .
 
+# --locked asserts that uv.lock is in sync with pyproject.toml, so an image
+# can never be built from a lockfile that drifted.
+RUN uv sync --locked
 
-# Install the project according to pyproject.toml + uv.lock
-# --locked asserts that uv.lock is in sync with pyproject.toml
-RUN uv sync
-
-# Create separate venv for csvkit to avoid dependency conflicts
-RUN uv venv $VENV --python 3.12 && \
-    uv pip install --python $VENV/bin/python csvkit duckdb-cli
+# duckdb, for the filtering workflows in the docs: export obs to CSV, query it,
+# feed the names back to `adata subset --obs`. A single static binary, so it
+# needs no venv and cannot conflict with the project's dependencies.
+ARG DUCKDB_VERSION=v1.1.3
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl unzip \
+    && ARCH="$(dpkg --print-architecture)" \
+    && case "$ARCH" in \
+         amd64) DUCKDB_ARCH=amd64 ;; \
+         arm64) DUCKDB_ARCH=aarch64 ;; \
+         *) echo "unsupported architecture: $ARCH" >&2; exit 1 ;; \
+       esac \
+    && curl -fsSL -o /tmp/duckdb.zip \
+         "https://github.com/duckdb/duckdb/releases/download/${DUCKDB_VERSION}/duckdb_cli-linux-${DUCKDB_ARCH}.zip" \
+    && unzip -q /tmp/duckdb.zip -d /usr/local/bin \
+    && chmod +x /usr/local/bin/duckdb \
+    && rm /tmp/duckdb.zip \
+    && apt-get purge -y curl unzip \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
 
-# Put the project venv on PATH so `h5ad` is directly runnable
-ENV PATH="/cli/.venv/bin:${VENV}/bin:${PATH}"
+# Put the project venv on PATH so `adata` is directly runnable
+ENV PATH="/cli/.venv/bin:${PATH}"
 
-# Default entrypoint: run the CLI
-ENTRYPOINT ["h5ad"]
+ENTRYPOINT ["adata"]
 CMD ["--help"]
-