From 76b9c0f2d2e27b027d68bc110e2785ee4b0272fb Mon Sep 17 00:00:00 2001 From: Aljes Date: Mon, 23 Mar 2026 12:50:31 +0000 Subject: [PATCH 01/23] Bump version to 0.3.1 in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 281812b..b10719e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "h5ad" -version = "0.2.0" +version = "0.3.1" description = "Streaming CLI utilities for exploring large AnnData .h5ad files" readme = "README.md" requires-python = ">=3.12" From 2bce36610fac96b485e7a9b7cec49dde7f5fc30f Mon Sep 17 00:00:00 2001 From: Aljes Date: Mon, 23 Mar 2026 16:10:39 +0000 Subject: [PATCH 02/23] Add function to ensure optional anndata groups in subset operations --- src/h5ad/core/subset.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/h5ad/core/subset.py b/src/h5ad/core/subset.py index d9e7829..da66a4b 100644 --- a/src/h5ad/core/subset.py +++ b/src/h5ad/core/subset.py @@ -44,6 +44,11 @@ def _group_get(parent: Any, key: str) -> Any | None: return parent[key] if key in parent else None +def _ensure_optional_anndata_groups(dst: Any) -> None: + for key in ("layers", "obsm", "obsp", "varm", "varp"): + _ensure_group(dst, key) + + def _decode_attr(value: Any) -> Any: if isinstance(value, bytes): return value.decode("utf-8") @@ -517,6 +522,8 @@ def subset_h5ad( total=1, ) + _ensure_optional_anndata_groups(dst) + if inplace: if file.exists(): if file.is_dir(): From 8553664f4ed9d24205164891fd0c66291dc140d9 Mon Sep 17 00:00:00 2001 From: Aljes Date: Mon, 23 Mar 2026 16:10:45 +0000 Subject: [PATCH 03/23] Add functions to ensure and validate AnnData root attributes in store operations --- src/h5ad/storage/__init__.py | 70 ++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/src/h5ad/storage/__init__.py b/src/h5ad/storage/__init__.py index 43d876d..6998344 100644 --- a/src/h5ad/storage/__init__.py +++ b/src/h5ad/storage/__init__.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any, Iterable, Optional, Sequence import shutil +import warnings import h5py @@ -15,6 +16,10 @@ import numpy as np +ROOT_ENCODING_TYPE = "anndata" +ROOT_ENCODING_VERSION = "0.1.0" + + @dataclass class Store: backend: str @@ -96,19 +101,63 @@ def open_store(path: Path, mode: str) -> Store: if backend == "zarr": _require_zarr() root = zarr.open_group(str(path), mode=mode) + if _is_writable_mode(mode): + ensure_anndata_root_attrs(root) + else: + warn_if_missing_anndata_root_attrs(root, path=path) return Store(backend="zarr", root=root, path=path) root = h5py.File(path, mode) + if _is_writable_mode(mode): + ensure_anndata_root_attrs(root) + else: + warn_if_missing_anndata_root_attrs(root, path=path) return Store(backend="hdf5", root=root, path=path) +def _decode_attr(value: Any) -> Any: + if isinstance(value, bytes): + return value.decode("utf-8") + return value + + +def _is_writable_mode(mode: str) -> bool: + return any(flag in mode for flag in ("w", "a", "+", "x")) + + +def has_valid_anndata_root_attrs(root: Any) -> bool: + enc_type = _decode_attr(root.attrs.get("encoding-type", None)) + enc_ver = _decode_attr(root.attrs.get("encoding-version", None)) + return enc_type == ROOT_ENCODING_TYPE and enc_ver == ROOT_ENCODING_VERSION + + +def ensure_anndata_root_attrs(root: Any) -> None: + root.attrs["encoding-type"] = ROOT_ENCODING_TYPE + root.attrs["encoding-version"] = ROOT_ENCODING_VERSION + + +def warn_if_missing_anndata_root_attrs(root: Any, *, path: Path) -> None: + if has_valid_anndata_root_attrs(root): + return + + enc_type = _decode_attr(root.attrs.get("encoding-type", None)) + enc_ver = _decode_attr(root.attrs.get("encoding-version", None)) + warnings.warn( + ( + f"Store '{path}' root is missing required AnnData attrs " + f"(encoding-type='anndata', encoding-version='0.1.0'). " + f"Found encoding-type={enc_type!r}, encoding-version={enc_ver!r}." + ), + UserWarning, + stacklevel=2, + ) + + def _normalize_attr_value(value: Any, target_backend: str) -> Any: if target_backend == "zarr": if isinstance(value, bytes): return value.decode("utf-8") if isinstance(value, (list, tuple)): - return [ - v.decode("utf-8") if isinstance(v, bytes) else v for v in value - ] + return [v.decode("utf-8") if isinstance(v, bytes) else v for v in value] if isinstance(value, np.ndarray): if value.dtype.kind in ("S", "O"): return [ @@ -187,7 +236,9 @@ def create_dataset( if zarr_format == 3: kwargs = dict(kwargs) kwargs.pop("compressor", None) - elif zarr_format == 2 and "compressors" in kwargs and "compressor" not in kwargs: + elif ( + zarr_format == 2 and "compressors" in kwargs and "compressor" not in kwargs + ): kwargs = dict(kwargs) compressors = kwargs.pop("compressors") if isinstance(compressors, (list, tuple)) and len(compressors) == 1: @@ -234,8 +285,12 @@ def copy_dataset(src: Any, dst_group: Any, name: str) -> Any: return ds -def copy_tree(src_obj: Any, dst_group: Any, name: str, *, exclude: Iterable[str] = ()) -> Any: - if is_hdf5_group(dst_group) and (is_hdf5_group(src_obj) or is_hdf5_dataset(src_obj)): +def copy_tree( + src_obj: Any, dst_group: Any, name: str, *, exclude: Iterable[str] = () +) -> Any: + if is_hdf5_group(dst_group) and ( + is_hdf5_group(src_obj) or is_hdf5_dataset(src_obj) + ): if not exclude: dst_group.copy(src_obj, dst_group, name) return dst_group[name] @@ -256,6 +311,9 @@ def copy_tree(src_obj: Any, dst_group: Any, name: str, *, exclude: Iterable[str] def copy_store_contents(src_root: Any, dst_root: Any) -> None: + target_backend = "zarr" if is_zarr_group(dst_root) else "hdf5" + copy_attrs(src_root.attrs, dst_root.attrs, target_backend=target_backend) + ensure_anndata_root_attrs(dst_root) for key in src_root.keys(): copy_tree(src_root[key], dst_root, key) From 796e74cfaee49da8c8facc1c4e43fcdefa2928f6 Mon Sep 17 00:00:00 2001 From: Aljes Date: Mon, 23 Mar 2026 16:10:51 +0000 Subject: [PATCH 04/23] Add tests for AnnData root encoding attributes enforcement and warnings --- tests/test_storage_root_attrs.py | 42 ++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_storage_root_attrs.py diff --git a/tests/test_storage_root_attrs.py b/tests/test_storage_root_attrs.py new file mode 100644 index 0000000..13f4621 --- /dev/null +++ b/tests/test_storage_root_attrs.py @@ -0,0 +1,42 @@ +"""Tests for AnnData root encoding attributes enforcement/warnings.""" + +from pathlib import Path + +import h5py +import pytest + +from h5ad.storage import open_store + + +def _make_minimal_h5ad(path: Path) -> None: + with h5py.File(path, "w") as f: + obs = f.create_group("obs") + obs.attrs["_index"] = "obs_names" + obs.create_dataset("obs_names", data=[b"cell_1"]) + + var = f.create_group("var") + var.attrs["_index"] = "var_names" + var.create_dataset("var_names", data=[b"gene_1"]) + + f.create_dataset("X", data=[[1.0]]) + + +def test_open_store_read_warns_for_missing_root_attrs(temp_dir: Path) -> None: + file_path = temp_dir / "missing_root_attrs.h5ad" + _make_minimal_h5ad(file_path) + + with pytest.warns(UserWarning, match="missing required AnnData attrs"): + with open_store(file_path, "r"): + pass + + +def test_open_store_writable_mode_sets_root_attrs(temp_dir: Path) -> None: + file_path = temp_dir / "set_root_attrs.h5ad" + _make_minimal_h5ad(file_path) + + with open_store(file_path, "a"): + pass + + with h5py.File(file_path, "r") as f: + assert f.attrs.get("encoding-type") == "anndata" + assert f.attrs.get("encoding-version") == "0.1.0" From 4d44e4639c9a225a58ba364e746ea72826ff50e6 Mon Sep 17 00:00:00 2001 From: Aljes Date: Mon, 23 Mar 2026 16:10:57 +0000 Subject: [PATCH 05/23] Add test for optional empty groups in subset_h5ad function --- tests/test_subset.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_subset.py b/tests/test_subset.py index 78c5cf8..a21fc20 100644 --- a/tests/test_subset.py +++ b/tests/test_subset.py @@ -284,6 +284,28 @@ def test_subset_sparse_empty_result(self, sample_sparse_csr_h5ad, temp_dir): class TestSubsetH5ad: """Integration tests for subset_h5ad function.""" + def test_subset_h5ad_creates_optional_empty_groups(self, sample_h5ad_file, temp_dir): + """Subset output should include optional AnnData groups even if absent in source.""" + obs_file = temp_dir / "obs_names.txt" + obs_file.write_text("cell_1\ncell_3\n") + + output = temp_dir / "subset.h5ad" + console = Console(stderr=True) + + subset_h5ad( + file=sample_h5ad_file, + output=output, + obs_file=obs_file, + var_file=None, + chunk_rows=1024, + console=console, + ) + + with h5py.File(output, "r") as f: + for key in ("layers", "obsm", "obsp", "varm", "varp"): + assert key in f + assert isinstance(f[key], h5py.Group) + def test_subset_h5ad_obs_only(self, sample_h5ad_file, temp_dir): """Test subsetting h5ad file by obs only.""" obs_file = temp_dir / "obs_names.txt" From 1911cdc7652e194be573865ec5326742b84a3976 Mon Sep 17 00:00:00 2001 From: Aljes Date: Mon, 23 Mar 2026 16:11:02 +0000 Subject: [PATCH 06/23] Refactor test_subset_h5ad to improve readability and ensure optional empty groups are included in the subset output --- tests/test_subset.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_subset.py b/tests/test_subset.py index a21fc20..42d5ec6 100644 --- a/tests/test_subset.py +++ b/tests/test_subset.py @@ -284,7 +284,9 @@ def test_subset_sparse_empty_result(self, sample_sparse_csr_h5ad, temp_dir): class TestSubsetH5ad: """Integration tests for subset_h5ad function.""" - def test_subset_h5ad_creates_optional_empty_groups(self, sample_h5ad_file, temp_dir): + def test_subset_h5ad_creates_optional_empty_groups( + self, sample_h5ad_file, temp_dir + ): """Subset output should include optional AnnData groups even if absent in source.""" obs_file = temp_dir / "obs_names.txt" obs_file.write_text("cell_1\ncell_3\n") @@ -460,7 +462,9 @@ def test_subset_h5ad_obsp_sparse_group(self, temp_dir): conn.attrs["shape"] = np.array([4, 4], dtype=np.int64) conn.create_dataset("data", data=np.array([1.0, 2.0, 3.0, 4.0])) conn.create_dataset("indices", data=np.array([0, 1, 2, 3], dtype=np.int64)) - conn.create_dataset("indptr", data=np.array([0, 1, 2, 3, 4], dtype=np.int64)) + conn.create_dataset( + "indptr", data=np.array([0, 1, 2, 3, 4], dtype=np.int64) + ) obs_file = temp_dir / "obs_names.txt" obs_file.write_text("cell_1\ncell_3\n") @@ -584,7 +588,8 @@ def _csr_group(parent, name, shape): obs = f.create_group("obs") obs.attrs["_index"] = "obs_names" obs.create_dataset( - "obs_names", data=np.array(["cell_1", "cell_2", "cell_3", "cell_4"], dtype="S") + "obs_names", + data=np.array(["cell_1", "cell_2", "cell_3", "cell_4"], dtype="S"), ) var = f.create_group("var") From a62f6e0d437ae7937d973216ba0a83e27d04398f Mon Sep 17 00:00:00 2001 From: Aljes Date: Mon, 23 Mar 2026 16:11:20 +0000 Subject: [PATCH 07/23] Bump h5ad package version to 0.3.1 --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index cb342bd..7884068 100644 --- a/uv.lock +++ b/uv.lock @@ -134,7 +134,7 @@ wheels = [ [[package]] name = "h5ad" -version = "0.3.0" +version = "0.3.1" source = { editable = "." } dependencies = [ { name = "h5py" }, From 939aec853fd7fd2f47e1c7467a123b138ecbc0d2 Mon Sep 17 00:00:00 2001 From: Aljes Date: Mon, 23 Mar 2026 16:23:19 +0000 Subject: [PATCH 08/23] Refactor documentation for mappings in HDF5 and Zarr formats to improve clarity and consistency --- docs/ELEMENTS_h5ad.md | 32 ++++++++++++++++++++++---------- docs/ELEMENTS_zarr.md | 32 ++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/docs/ELEMENTS_h5ad.md b/docs/ELEMENTS_h5ad.md index acb491d..2cdd8db 100644 --- a/docs/ELEMENTS_h5ad.md +++ b/docs/ELEMENTS_h5ad.md @@ -18,7 +18,8 @@ It is intended to be GitHub-renderable Markdown (no Sphinx/MyST directives). - [DataFrame v0.2.0](#dataframe-v020) - [DataFrame v0.1.0 (legacy: anndata 0.7.x)](#dataframe-v010-legacy-anndata-07x) - [Legacy categorical columns (Series-level)](#legacy-categorical-columns-series-level) -- [Mappings / dict](#mappings--dict) +- [Mappings](#mappings) + - [Legacy mapping encoding (`dict` v0.1.0)](#legacy-mapping-encoding-dict-v010) - [Scalars](#scalars) - [Categorical arrays](#categorical-arrays) - [String arrays](#string-arrays) @@ -152,19 +153,28 @@ In v0.1.0 DataFrames, a categorical column dataset (e.g. `obs/cell_type`) can be - `categories`: an **HDF5 object reference** pointing to the corresponding `__categories/` dataset. -## Mappings / dict +## Mappings -### `encoding-type: dict`, `encoding-version: 0.1.0` +Mappings are stored as HDF5 **groups** on disk. -- A mapping **MUST** be stored as an HDF5 **group**. -- Group attributes: - - `encoding-type: "dict"` - - `encoding-version: "0.1.0"` -- Each entry in the group is another element (recursively). +- This includes standard AnnData mappings such as `layers`, `obsm`, `varm`, `obsp`, `varp`, and `uns`. +- Mappings are distinct from DataFrames and sparse arrays and do not require special mapping-specific attributes. +- Mapping semantics are recursive: entries in `uns` can themselves be groups containing additional encoded elements. -> **Legacy note** +> **Legacy compatibility note** > -> In anndata 0.7.x, groups used as mappings often had **no special attributes**. +> In earlier conventions (commonly seen in older docs and some files), mappings could carry +> `encoding-type: "dict"` and `encoding-version: "0.1.0"`. +> Readers should still accept this legacy metadata when encountered. + +### Legacy mapping encoding (`dict` v0.1.0) + +For backward compatibility, older files may encode mappings with explicit mapping metadata: + +- `encoding-type: "dict"` +- `encoding-version: "0.1.0"` + +This historical convention existed in earlier AnnData docs and files and should still be accepted by readers. ## Scalars @@ -172,6 +182,8 @@ In v0.1.0 DataFrames, a categorical column dataset (e.g. `obs/cell_type`) can be Scalars are stored as **0-dimensional datasets**. +These should typically only occur inside `uns` and are commonly used for saved parameters. + - Numeric scalars: - `encoding-type: "numeric-scalar"` - `encoding-version: "0.2.0"` diff --git a/docs/ELEMENTS_zarr.md b/docs/ELEMENTS_zarr.md index ce309e6..f547024 100644 --- a/docs/ELEMENTS_zarr.md +++ b/docs/ELEMENTS_zarr.md @@ -18,7 +18,8 @@ It is intended to be GitHub-renderable Markdown (no Sphinx/MyST directives). - [DataFrame v0.2.0](#dataframe-v020) - [DataFrame v0.1.0 (legacy: anndata 0.7.x)](#dataframe-v010-legacy-anndata-07x) - [Legacy categorical columns (Series-level)](#legacy-categorical-columns-series-level) -- [Mappings / dict](#mappings--dict) +- [Mappings](#mappings) + - [Legacy mapping encoding (`dict` v0.1.0)](#legacy-mapping-encoding-dict-v010) - [Scalars](#scalars) - [Categorical arrays](#categorical-arrays) - [String arrays](#string-arrays) @@ -154,19 +155,28 @@ In v0.1.0 DataFrames, a categorical column array (e.g. `obs/cell_type`) can be i (This differs from HDF5, which can store an object reference.) -## Mappings / dict +## Mappings -### `encoding-type: dict`, `encoding-version: 0.1.0` +Mappings are stored as Zarr **groups** on disk. -- A mapping **MUST** be stored as a Zarr **group**. -- Group attributes: - - `encoding-type: "dict"` - - `encoding-version: "0.1.0"` -- Each entry in the group is another element (recursively). +- This includes standard AnnData mappings such as `layers`, `obsm`, `varm`, `obsp`, `varp`, and `uns`. +- Mappings are distinct from DataFrames and sparse arrays and do not require special mapping-specific attributes. +- Mapping semantics are recursive: entries in `uns` can themselves be groups containing additional encoded elements. -> **Legacy note** +> **Legacy compatibility note** > -> In anndata 0.7.x, groups used as mappings often had **no special attributes**. +> In earlier conventions (commonly seen in older docs and some files), mappings could carry +> `encoding-type: "dict"` and `encoding-version: "0.1.0"`. +> Readers should still accept this legacy metadata when encountered. + +### Legacy mapping encoding (`dict` v0.1.0) + +For backward compatibility, older files may encode mappings with explicit mapping metadata: + +- `encoding-type: "dict"` +- `encoding-version: "0.1.0"` + +This historical convention existed in earlier AnnData docs and files and should still be accepted by readers. ## Scalars @@ -174,6 +184,8 @@ In v0.1.0 DataFrames, a categorical column array (e.g. `obs/cell_type`) can be i Scalars are stored as **0-dimensional Zarr arrays**. +These should typically only occur inside `uns` and are commonly used for saved parameters. + - Numeric scalars: - `encoding-type: "numeric-scalar"` - `encoding-version: "0.2.0"` From 7ba61c9ca538d3cf412e82cabd2a72d189841685 Mon Sep 17 00:00:00 2001 From: Aljes Binkevich Date: Mon, 23 Mar 2026 16:51:29 +0000 Subject: [PATCH 09/23] Update src/h5ad/storage/__init__.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/h5ad/storage/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/h5ad/storage/__init__.py b/src/h5ad/storage/__init__.py index 6998344..29c2227 100644 --- a/src/h5ad/storage/__init__.py +++ b/src/h5ad/storage/__init__.py @@ -143,8 +143,8 @@ def warn_if_missing_anndata_root_attrs(root: Any, *, path: Path) -> None: enc_ver = _decode_attr(root.attrs.get("encoding-version", None)) warnings.warn( ( - f"Store '{path}' root is missing required AnnData attrs " - f"(encoding-type='anndata', encoding-version='0.1.0'). " + f"Store '{path}' root has missing or invalid AnnData attrs " + f"(encoding-type={ROOT_ENCODING_TYPE!r}, encoding-version={ROOT_ENCODING_VERSION!r}). " f"Found encoding-type={enc_type!r}, encoding-version={enc_ver!r}." ), UserWarning, From ff56bf071afbb8fa8cc6859e2280999fccd7c3b5 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 12:09:27 +0100 Subject: [PATCH 10/23] Rename to adata-cli and restore AnnData format compatibility The CLI could not read any file written by anndata >= 0.11. Since pandas' future.infer_string became the default, anndata writes obs/_index as a nullable-string-array *group* rather than a dataset, and axis_len required a dataset, so view, export dataframe and subset all failed outright on every recent file. HDF5 -> Zarr conversion was also broken on any store with string columns, and subset silently dropped raw/. Rename the project to adata-cli (package src/adata, command `adata`) and rename `info` to `view`; both old names remain as deprecating aliases through 0.x. Add src/adata/elements/ as the single home for on-disk format knowledge -- spec (encoding constants), strings (backend dtype reconciliation), read (every historical layout) and write (current spec only). The import, export and subset paths now share it instead of each reimplementing the format. Fixes: - resolve_index/element_len accept a group-valued index, so modern files read - create_dataset resolves text to variable-length UTF-8 per backend, so all four HDF5/Zarr pairings convert with categoricals and nullable dtypes intact - subset_axis_group narrows group-valued columns instead of copying them whole, which had been producing stores with mismatched column lengths - raw/ is subset against its own var axis; unrecognised top-level keys are copied with a warning rather than dropped - every written element is tagged; None round-trips via anndata's `null` encoding instead of an invented marker attribute - categoricals survive a CSV round-trip - sparse subsetting streams in blocks (byte-exact against scipy, CSR and CSC) - get_entry_type dispatches on encoding-type before falling back to structure - column-order is honoured on export New: `ls` for any HDF5/Zarr store including .loom; dataframe export from any path; results on stdout with status on stderr; npy export to stdout. tests/test_anndata_roundtrip.py has anndata write the fixtures and read the results back, with OldFormatWarning promoted to an error. The CI matrix runs tests/ on 3.12 and 3.13 rather than naming files individually, which is why test_storage_root_attrs.py had never run. Co-Authored-By: Claude Opus 5 --- .github/workflows/tests.yml | 31 +- .gitignore | 11 + README.md | 27 +- docs/GET_STARTED.md | 28 +- pyproject.toml | 18 +- src/{h5ad => adata}/__init__.py | 0 src/{h5ad => adata}/cli.py | 177 ++++++-- src/adata/commands/__init__.py | 5 + src/{h5ad => adata}/commands/export.py | 18 +- src/{h5ad => adata}/commands/import_data.py | 10 +- src/{h5ad => adata}/commands/info.py | 21 +- src/adata/commands/ls.py | 114 +++++ src/{h5ad => adata}/commands/subset.py | 2 +- src/{h5ad => adata}/core/__init__.py | 0 src/adata/core/info.py | 276 ++++++++++++ src/adata/core/read.py | 28 ++ src/{h5ad => adata}/core/subset.py | 444 ++++++++++++++------ src/adata/elements/__init__.py | 10 + src/adata/elements/read.py | 310 ++++++++++++++ src/adata/elements/spec.py | 107 +++++ src/adata/elements/strings.py | 90 ++++ src/adata/elements/write.py | 246 +++++++++++ src/{h5ad => adata}/formats/__init__.py | 0 src/{h5ad => adata}/formats/array.py | 30 +- src/{h5ad => adata}/formats/common.py | 22 +- src/adata/formats/dataframe.py | 232 ++++++++++ src/{h5ad => adata}/formats/image.py | 4 +- src/{h5ad => adata}/formats/json_data.py | 84 ++-- src/{h5ad => adata}/formats/sparse.py | 27 +- src/{h5ad => adata}/formats/validate.py | 4 +- src/adata/info.py | 3 + src/adata/read.py | 19 + src/{h5ad => adata}/storage/__init__.py | 148 ++++++- src/{h5ad => adata}/util/__init__.py | 0 src/{h5ad => adata}/util/path.py | 0 src/h5ad/commands/__init__.py | 4 - src/h5ad/core/info.py | 221 ---------- src/h5ad/core/read.py | 142 ------- src/h5ad/formats/dataframe.py | 169 -------- src/h5ad/info.py | 3 - src/h5ad/read.py | 3 - tests/test_anndata_roundtrip.py | 283 +++++++++++++ tests/test_cli.py | 70 +-- tests/test_export.py | 4 +- tests/test_import.py | 2 +- tests/test_info_read.py | 6 +- tests/test_storage_root_attrs.py | 4 +- tests/test_subset.py | 2 +- tests/test_zarr.py | 6 +- uv.lock | 438 +++++++++++++++++-- 50 files changed, 2940 insertions(+), 963 deletions(-) create mode 100644 .gitignore rename src/{h5ad => adata}/__init__.py (100%) rename src/{h5ad => adata}/cli.py (77%) create mode 100644 src/adata/commands/__init__.py rename src/{h5ad => adata}/commands/export.py (79%) rename src/{h5ad => adata}/commands/import_data.py (92%) rename src/{h5ad => adata}/commands/info.py (89%) create mode 100644 src/adata/commands/ls.py rename src/{h5ad => adata}/commands/subset.py (90%) rename src/{h5ad => adata}/core/__init__.py (100%) create mode 100644 src/adata/core/info.py create mode 100644 src/adata/core/read.py rename src/{h5ad => adata}/core/subset.py (52%) create mode 100644 src/adata/elements/__init__.py create mode 100644 src/adata/elements/read.py create mode 100644 src/adata/elements/spec.py create mode 100644 src/adata/elements/strings.py create mode 100644 src/adata/elements/write.py rename src/{h5ad => adata}/formats/__init__.py (100%) rename src/{h5ad => adata}/formats/array.py (77%) rename src/{h5ad => adata}/formats/common.py (73%) create mode 100644 src/adata/formats/dataframe.py rename src/{h5ad => adata}/formats/image.py (94%) rename src/{h5ad => adata}/formats/json_data.py (66%) rename src/{h5ad => adata}/formats/sparse.py (92%) rename src/{h5ad => adata}/formats/validate.py (97%) create mode 100644 src/adata/info.py create mode 100644 src/adata/read.py rename src/{h5ad => adata}/storage/__init__.py (66%) rename src/{h5ad => adata}/util/__init__.py (100%) rename src/{h5ad => adata}/util/path.py (100%) delete mode 100644 src/h5ad/commands/__init__.py delete mode 100644 src/h5ad/core/info.py delete mode 100644 src/h5ad/core/read.py delete mode 100644 src/h5ad/formats/dataframe.py delete mode 100644 src/h5ad/info.py delete mode 100644 src/h5ad/read.py create mode 100644 tests/test_anndata_roundtrip.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dde3803..f12a9d1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,22 +22,9 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12"] # add "3.13" if you want - module: - - name: cli - tests: tests/test_cli.py - - name: export - tests: tests/test_export.py - - name: import - tests: tests/test_import.py - - name: info-read - tests: tests/test_info_read.py - - name: subset - tests: tests/test_subset.py - - name: zarr - tests: tests/test_zarr.py + python-version: ["3.12", "3.13"] - name: tests (${{ matrix.module.name }}) + name: tests (py${{ matrix.python-version }}) steps: - uses: actions/checkout@v4 @@ -57,29 +44,29 @@ jobs: - name: Run tests with coverage run: | - uv run pytest -v -W default ${{ matrix.module.tests }} \ - --cov=h5ad \ + uv run pytest -v -W default tests/ \ + --cov=adata \ --cov-report=term-missing \ --cov-report=xml \ --cov-report=html \ - --junitxml=pytest-results-${{ matrix.module.name }}.xml + --junitxml=pytest-results-py${{ matrix.python-version }}.xml - name: Publish test results summary uses: EnricoMi/publish-unit-test-result-action@v2 if: always() with: - files: pytest-results-${{ matrix.module.name }}.xml - check_name: Test Results (${{ matrix.module.name }}) + files: pytest-results-py${{ matrix.python-version }}.xml + check_name: Test Results (py${{ matrix.python-version }}) - name: Upload coverage artifacts uses: actions/upload-artifact@v4 if: always() with: - name: coverage-${{ matrix.module.name }} + name: coverage-py${{ matrix.python-version }} path: | coverage.xml htmlcov/ - pytest-results-${{ matrix.module.name }}.xml + pytest-results-py${{ matrix.python-version }}.xml retention-days: 30 - name: Upload coverage to Codecov diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f06a69c --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +__pycache__/ +*.py[cod] +.venv/ +build/ +dist/ +*.egg-info/ +.coverage +coverage.xml +htmlcov/ +.pytest_cache/ +pytest-results*.xml diff --git a/README.md b/README.md index 9b1f09f..ab893ad 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# h5ad CLI +# adata CLI A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) without loading them fully into memory. Streams data directly from disk for efficient inspection of structure, metadata, and matrices. @@ -7,14 +7,16 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with - Streaming access to very large `.h5ad` and `.zarr` stores - Auto-detects `.h5ad` files vs `.zarr` directories - Chunked processing for dense and sparse matrices (CSR/CSC) -- Rich terminal output with progress indicators +- Reads every AnnData on-disk layout, from 0.7.x through the current spec, and always writes the current one +- Converts between HDF5 and Zarr (v2 and v3) in either direction +- Rich terminal output with progress indicators, kept on stderr so results pipe cleanly ## Installation Using [uv](https://docs.astral.sh/uv/) (recommended): ```bash -git clone https://github.com/cellgeni/h5ad-cli.git -cd h5ad-cli +git clone https://github.com/cellgeni/adata-cli.git +cd adata-cli uv sync ``` @@ -25,8 +27,8 @@ uv sync --extra dev Alternative with pip: ```bash -git clone https://github.com/cellgeni/h5ad-cli.git -cd h5ad-cli +git clone https://github.com/cellgeni/adata-cli.git +cd adata-cli pip install . ``` @@ -35,23 +37,22 @@ For development and testing with pip: pip install -e ".[dev]" ``` -See [docs/TESTING.md](docs/TESTING.md) for testing documentation. - ## Commands (Overview) -Run help at any level (e.g. `uv run h5ad --help`, `uv run h5ad export --help`). +Run help at any level (e.g. `adata --help`, `adata export --help`). -- `info` – read-only inspection of store layout, shapes, and type hints; supports drilling into paths like `obsm/X_pca` or `uns`. +- `view` – AnnData-aware inspection: store layout, shapes, and encodings; supports drilling into paths like `obsm/X_pca` or `uns`. +- `ls` – list the contents of any HDF5 or Zarr store as a tree, with no AnnData assumptions (works on `.loom` and plain `.h5`); `-1` emits bare paths for piping. - `subset` – stream and write a filtered copy based on obs/var name lists, preserving dense and sparse matrix encodings. -- `export` – extract data from a store; subcommands: `dataframe` (obs/var to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). +- `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given. - `import` – write new data into a store; subcommands: `dataframe` (CSV → obs/var), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON). See [docs/GET_STARTED.md](docs/GET_STARTED.md) for a short tutorial. ## Docker -A docker image is available on QUAY: `quay.io/cellgeni/h5ad-cli:latest`. Pull and run with: +A docker image is available on QUAY: `quay.io/cellgeni/adata-cli:latest`. Pull and run with: ```bash -docker run --rm -it -v /path/to/data:/data quay.io/cellgeni/h5ad-cli:latest h5ad info /data/your_file.h5ad +docker run --rm -it -v /path/to/data:/data quay.io/cellgeni/adata-cli:latest adata view /data/your_file.h5ad ``` \ No newline at end of file diff --git a/docs/GET_STARTED.md b/docs/GET_STARTED.md index 35ccbb4..14d4811 100644 --- a/docs/GET_STARTED.md +++ b/docs/GET_STARTED.md @@ -6,15 +6,15 @@ This short walkthrough shows the basic workflow: inspect a store, export metadat Using uv (recommended): ```bash -git clone https://github.com/cellgeni/h5ad-cli.git -cd h5ad-cli +git clone https://github.com/cellgeni/adata-cli.git +cd adata-cli uv sync ``` With pip: ```bash -git clone https://github.com/cellgeni/h5ad-cli.git -cd h5ad-cli +git clone https://github.com/cellgeni/adata-cli.git +cd adata-cli pip install . ``` @@ -36,7 +36,7 @@ wget -O visium.h5ad https://exampledata.scverse.org/squidpy/figshare/visium_hne_ Now run `info` to see the file structure: ```bash -uv run h5ad info visium.h5ad +adata view visium.h5ad ``` ``` An object with n_obs × n_var: 2688 × 18078 @@ -52,7 +52,7 @@ pct_counts_in_top_50_genes, pct_counts_mt, total_counts, total_counts_mt To inspect a specific entry: ```bash -uv run h5ad info visium.h5ad obsm/X_pca +adata view visium.h5ad obsm/X_pca ``` ``` Path: obsm/X_pca @@ -66,7 +66,7 @@ Details: Dense matrix 2688×50 (float32) View the first few lines of the `obs` dataframe: ```bash -uv run h5ad export dataframe visium.h5ad obs --head 10 +adata export dataframe visium.h5ad obs --head 10 ``` ```csv _index,array_col,array_row,cluster,in_tissue,leiden,log1p_n_genes_by_counts,log1p_total_counts,log1p_total_counts_mt,n_counts,n_genes_by_counts,pct_counts_in_top_100_genes,pct_counts_in_top_200_genes,pct_counts_in_top_500_genes,pct_counts_in_top_50_genes,pct_counts_mt,total_counts,total_counts_mt @@ -84,7 +84,7 @@ AAACGGTTGCGAACTG-1,59,67,Lateral_ventricle,1,Striatum,8.718663567048953,10.25400 Export cell metadata to a CSV file: ```bash -uv run h5ad export dataframe visium.h5ad obs --output cells.csv +adata export dataframe visium.h5ad obs --output cells.csv wc -l cells.csv # 2689 cells.csv ``` @@ -121,12 +121,12 @@ wc -l barcodes.txt # 257 barcodes.txt Now you can use this list to create a subset `.h5ad` file: ```bash -uv run h5ad subset visium.h5ad --output cortex2.h5ad --obs barcodes.txt +adata subset visium.h5ad --output cortex2.h5ad --obs barcodes.txt ``` Check the result: ```bash -uv run h5ad info cortex2.h5ad +adata view cortex2.h5ad ``` ``` An object with n_obs × n_var: 257 × 18078 @@ -148,12 +148,12 @@ cut -d ',' -f 1-5 cells.csv > cells1to5.csv Now import it back into `cortex2.h5ad` with the `_index` column as index: ```bash -uv run h5ad import dataframe visium.h5ad obs cells1to5.csv --index-column _index --output visium_obs1to5.h5ad +adata import dataframe visium.h5ad obs cells1to5.csv --index-column _index --output visium_obs1to5.h5ad ``` Check the updated `obs` structure: ```bash -uv run h5ad info visium_obs1to5.h5ad +adata view visium_obs1to5.h5ad ``` ``` An object with n_obs × n_var: 2688 × 18078 @@ -169,12 +169,12 @@ pct_dropout_by_counts, total_counts, variances, variances_norm You can also import the data into existing file: ```bash -uv run h5ad import dataframe visium.h5ad obs cells1to5.csv --index-column _index --inplace +adata import dataframe visium.h5ad obs cells1to5.csv --index-column _index --inplace ``` Check the updated `obs` structure: ```bash -uv run h5ad info visium.h5ad +adata view visium.h5ad ``` ``` An object with n_obs × n_var: 2688 × 18078 diff --git a/pyproject.toml b/pyproject.toml index 6f1eb12..e90c4e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "h5ad" -version = "0.3.1" -description = "Streaming CLI utilities for exploring large AnnData .h5ad and .zarr files" +name = "adata-cli" +version = "0.4.0.dev0" +description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores" readme = "README.md" requires-python = ">=3.12" license = "MIT" @@ -47,17 +47,25 @@ Issues = "https://github.com/cellgeni/h5ad-cli/issues" dev = [ "pytest>=8.3.4", "pytest-cov>=6.0.0", + # Used only by tests/test_anndata_roundtrip.py, to check that what this + # tool reads and writes matches what anndata itself produces. + "anndata>=0.13", ] [build-system] requires = ["uv_build>=0.8.0,<0.9.0"] build-backend = "uv_build" +[tool.uv.build-backend] +module-name = "adata" + [project.scripts] -h5ad = "h5ad.cli:main" +adata = "adata.cli:main" +# Deprecated alias, removed in 1.0.0 +h5ad = "adata.cli:main_deprecated" [tool.coverage.run] -source = ["src/h5ad"] +source = ["src/adata"] omit = [ "*/tests/*", "*/__pycache__/*", diff --git a/src/h5ad/__init__.py b/src/adata/__init__.py similarity index 100% rename from src/h5ad/__init__.py rename to src/adata/__init__.py diff --git a/src/h5ad/cli.py b/src/adata/cli.py similarity index 77% rename from src/h5ad/cli.py rename to src/adata/cli.py index 66bbd22..4b556fc 100644 --- a/src/h5ad/cli.py +++ b/src/adata/cli.py @@ -1,4 +1,4 @@ -"""CLI for h5ad files with export and import subcommands.""" +"""CLI for AnnData stores (.h5ad / .zarr) with export and import subcommands.""" from pathlib import Path from typing import Optional, Sequence, List @@ -6,7 +6,8 @@ from rich.console import Console import typer -from h5ad.commands import ( +from adata.commands import ( + list_store, show_info, subset_h5ad, export_mtx, @@ -15,18 +16,21 @@ export_table, ) -from h5ad.commands import export_image as export_image_cmd +from adata.commands import export_image as export_image_cmd app = typer.Typer( - help="Streaming CLI for huge .h5ad and .zarr files (info, subset, export, import)." + help="Streaming CLI for huge AnnData .h5ad and .zarr stores " + "(view, ls, subset, export, import)." ) # Use stderr for status/progress to keep stdout clean for data output # force_terminal=True ensures Rich output is visible even in non-TTY environments console = Console(stderr=True, force_terminal=True) +# Results go to stdout so they can be piped; status and errors stay on stderr. +out_console = Console() # Create sub-apps for export and import -export_app = typer.Typer(help="Export objects from h5ad files.") -import_app = typer.Typer(help="Import objects into h5ad files.") +export_app = typer.Typer(help="Export objects from an AnnData store.") +import_app = typer.Typer(help="Import objects into an AnnData store.") app.add_typer(export_app, name="export") app.add_typer(import_app, name="import") @@ -34,8 +38,8 @@ # ============================================================================ # INFO command # ============================================================================ -@app.command() -def info( +@app.command("view") +def view( file: Path = typer.Argument( ..., help="Path to the .h5ad/.zarr store", @@ -62,18 +66,104 @@ def info( ), ) -> None: """ - Show high-level information about the .h5ad file. + Show high-level information about an AnnData store. Use --tree to see a tree of all entries. - Use --entry to inspect a specific entry in detail. + Pass an entry path to inspect a specific entry in detail. Examples: - h5ad info data.h5ad - h5ad info --tree data.h5ad - h5ad info obsm/X_pca data.h5ad + adata view data.h5ad + adata view --tree data.h5ad + adata view data.h5ad obsm/X_pca """ try: - show_info(file, console, show_types=tree, depth=depth, entry_path=entry) + show_info( + file, + console, + show_types=tree, + depth=depth, + entry_path=entry, + out_console=out_console, + ) + except Exception as e: + console.print(f"[bold red]Error:[/] {e}") + raise typer.Exit(code=1) + + +@app.command("info", hidden=True) +def info( + file: Path = typer.Argument( + ..., + help="Path to the .h5ad/.zarr store", + exists=True, + readable=True, + dir_okay=True, + file_okay=True, + ), + entry: Optional[str] = typer.Argument( + None, + help="Entry path to inspect (e.g., 'obsm/X_pca', 'X', 'uns')", + ), + tree: bool = typer.Option(False, "--tree", "-t", help="Show a tree of all entries"), + depth: int = typer.Option( + None, "--depth", "-d", help="Maximum recursion depth for tree display" + ), +) -> None: + """Deprecated alias for 'view'. Removed in 1.0.0.""" + console.print( + "[yellow]Warning:[/] 'info' is deprecated and will be removed in 1.0.0; " + "use 'view' instead.", + ) + view(file=file, entry=entry, tree=tree, depth=depth) + + +# ============================================================================ +# LS command +# ============================================================================ +@app.command("ls") +def ls( + file: Path = typer.Argument( + ..., + help="Path to any .h5ad/.zarr/.h5 store", + exists=True, + readable=True, + dir_okay=True, + file_okay=True, + ), + entry: Optional[str] = typer.Argument( + None, help="Only list below this path (e.g. 'obsm', 'uns/spatial')" + ), + depth: Optional[int] = typer.Option( + None, "--depth", "-d", help="Maximum depth to descend" + ), + long: bool = typer.Option( + False, "--long", "-l", help="Show type, shape, dtype and encoding" + ), + plain: bool = typer.Option( + False, "-1", "--plain", help="One bare path per line, for piping" + ), +) -> None: + """ + List the contents of any HDF5 or Zarr store. + + Makes no AnnData assumptions, so it also works on .loom and plain .h5 + files. Use -1 to pipe paths into other tools. + + Examples: + adata ls data.h5ad + adata ls data.h5ad --long + adata ls data.h5ad obsm --depth 1 + adata ls data.h5ad -1 | grep spatial + """ + try: + list_store( + file, + out_console, + entry_path=entry, + depth=depth, + long=long, + plain=plain, + ) except Exception as e: console.print(f"[bold red]Error:[/] {e}") raise typer.Exit(code=1) @@ -128,7 +218,7 @@ def subset( help="Row chunk size for dense matrices", ), ) -> None: - """Subset an h5ad by obs and/or var names.""" + """Subset an AnnData store by obs and/or var names.""" if obs is None and var is None: console.print( "[bold red]Error:[/] At least one of --obs or --var must be provided.", @@ -170,7 +260,9 @@ def export_dataframe( dir_okay=True, file_okay=True, ), - entry: str = typer.Argument(..., help="Entry path to export ('obs' or 'var')"), + entry: str = typer.Argument( + ..., help="Path of the dataframe to export (e.g. 'obs', 'var', 'raw/var')" + ), output: Path = typer.Option( None, "--output", "-o", writable=True, help="Output CSV file path" ), @@ -193,20 +285,17 @@ def export_dataframe( ), ) -> None: """ - Export a dataframe (obs or var) to CSV. + Export any dataframe-encoded group to CSV. + + Not limited to obs/var -- any path holding a dataframe works, including + ones under raw/, obsm/ or uns/. Examples: - h5ad export dataframe data.h5ad obs --output obs.csv - h5ad export dataframe data.h5ad var --output var.csv --columns gene_id,mean - h5ad export dataframe data.h5ad obs --head 100 + adata export dataframe data.h5ad obs --output obs.csv + adata export dataframe data.h5ad var --columns gene_id,mean + adata export dataframe data.h5ad raw/var --head 100 """ - if entry not in ("obs", "var"): - console.print( - f"[bold red]Error:[/] Dataframe export is only supported for 'obs' or 'var' at this point, not '{entry}'.", - ) - raise typer.Exit(code=1) - col_list: Optional[List[str]] = None if columns: col_list = [col.strip() for col in columns.split(",") if col.strip()] @@ -239,8 +328,12 @@ def export_array( entry: str = typer.Argument( ..., help="Entry path to export (e.g., 'obsm/X_pca', 'varm/PCs', 'X')" ), - output: Path = typer.Option( - ..., "--output", "-o", help="Output .npy file path", writable=True + output: Optional[Path] = typer.Option( + None, + "--output", + "-o", + help="Output .npy file path (defaults to stdout)", + writable=True, ), chunk_elements: int = typer.Option( 100_000, @@ -252,10 +345,13 @@ def export_array( """ Export a dense array or matrix to NumPy .npy format. + Writes to stdout when no --output is given. Writing to a file streams in + chunks; stdout is not seekable, so that path holds the array in memory. + Examples: - h5ad export array data.h5ad obsm/X_pca pca.npy - h5ad export array data.h5ad X matrix.npy - h5ad export array data.h5ad varm/PCs loadings.npy + adata export array data.h5ad obsm/X_pca -o pca.npy + adata export array data.h5ad X -o matrix.npy + adata export array data.h5ad obsm/X_umap > umap.npy """ try: @@ -416,7 +512,7 @@ def export_image( # ============================================================================ def _get_target_file(file: Path, output: Optional[Path], inplace: bool) -> Path: """Determine target path and copy/convert if needed.""" - from h5ad.commands.import_data import _prepare_target_path + from adata.commands.import_data import _prepare_target_path return _prepare_target_path(file, output, inplace, console) @@ -464,7 +560,7 @@ def import_dataframe( h5ad import dataframe data.h5ad obs cells.csv -o output.h5ad -i cell_id h5ad import dataframe data.h5ad var genes.csv --inplace -i gene_id """ - from h5ad.commands.import_data import _import_csv + from adata.commands.import_data import _import_csv if entry not in ("obs", "var"): console.print( @@ -526,7 +622,7 @@ def import_array( h5ad import array data.h5ad obsm/X_pca pca.npy -o output.h5ad h5ad import array data.h5ad X matrix.npy --inplace """ - from h5ad.commands.import_data import _import_npy + from adata.commands.import_data import _import_npy if not inplace and output is None: console.print( @@ -582,7 +678,7 @@ def import_sparse( h5ad import sparse data.h5ad X matrix.mtx -o output.h5ad h5ad import sparse data.h5ad layers/counts counts.mtx --inplace """ - from h5ad.commands.import_data import _import_mtx + from adata.commands.import_data import _import_mtx if not inplace and output is None: console.print( @@ -636,7 +732,7 @@ def import_dict( h5ad import dict data.h5ad uns/metadata config.json -o output.h5ad h5ad import dict data.h5ad uns settings.json --inplace """ - from h5ad.commands.import_data import _import_json + from adata.commands.import_data import _import_json if not inplace and output is None: console.print( @@ -655,3 +751,12 @@ def import_dict( def main(argv: Optional[Sequence[str]] = None) -> None: app(standalone_mode=True) + + +def main_deprecated(argv: Optional[Sequence[str]] = None) -> None: + """Entry point for the deprecated `h5ad` command name.""" + console.print( + "[yellow]Warning:[/] the 'h5ad' command is deprecated and will be removed " + "in 1.0.0; use 'adata' instead.", + ) + main(argv) diff --git a/src/adata/commands/__init__.py b/src/adata/commands/__init__.py new file mode 100644 index 0000000..2f15de5 --- /dev/null +++ b/src/adata/commands/__init__.py @@ -0,0 +1,5 @@ +from adata.commands.info import show_info +from adata.commands.subset import subset_h5ad +from adata.commands.export import export_table, export_image, export_json, export_mtx, export_npy +from adata.commands.import_data import import_object +from adata.commands.ls import list_store diff --git a/src/h5ad/commands/export.py b/src/adata/commands/export.py similarity index 79% rename from src/h5ad/commands/export.py rename to src/adata/commands/export.py index 22221a7..8bf7466 100644 --- a/src/h5ad/commands/export.py +++ b/src/adata/commands/export.py @@ -5,13 +5,12 @@ from rich.console import Console -from h5ad.formats.array import export_npy as export_npy_format -from h5ad.formats.common import EXPORTABLE_TYPES, IMAGE_EXTENSIONS, TYPE_EXTENSIONS -from h5ad.formats.dataframe import export_dataframe -from h5ad.formats.image import export_image as export_image_format -from h5ad.formats.json_data import export_json as export_json_format -from h5ad.formats.sparse import export_mtx as export_mtx_format -from h5ad.storage import open_store +from adata.formats.array import export_npy as export_npy_format +from adata.formats.dataframe import export_dataframe +from adata.formats.image import export_image as export_image_format +from adata.formats.json_data import export_json as export_json_format +from adata.formats.sparse import export_mtx as export_mtx_format +from adata.storage import open_store def export_table( @@ -38,7 +37,7 @@ def export_table( def export_npy( file: Path, obj: str, - out: Path, + out: Optional[Path], chunk_elements: int, console: Console, ) -> None: @@ -98,9 +97,6 @@ def export_image(file: Path, obj: str, out: Path, console: Console) -> None: __all__ = [ - "EXPORTABLE_TYPES", - "IMAGE_EXTENSIONS", - "TYPE_EXTENSIONS", "export_image", "export_json", "export_mtx", diff --git a/src/h5ad/commands/import_data.py b/src/adata/commands/import_data.py similarity index 92% rename from src/h5ad/commands/import_data.py rename to src/adata/commands/import_data.py index dad838a..b47c330 100644 --- a/src/h5ad/commands/import_data.py +++ b/src/adata/commands/import_data.py @@ -7,11 +7,11 @@ from rich.console import Console -from h5ad.formats.array import import_npy -from h5ad.formats.dataframe import import_dataframe -from h5ad.formats.json_data import import_json -from h5ad.formats.sparse import import_mtx -from h5ad.storage import copy_path, copy_store_contents, detect_backend, open_store +from adata.formats.array import import_npy +from adata.formats.dataframe import import_dataframe +from adata.formats.json_data import import_json +from adata.formats.sparse import import_mtx +from adata.storage import copy_path, copy_store_contents, detect_backend, open_store EXTENSION_FORMAT = { diff --git a/src/h5ad/commands/info.py b/src/adata/commands/info.py similarity index 89% rename from src/h5ad/commands/info.py rename to src/adata/commands/info.py index 76b56da..81e67b7 100644 --- a/src/h5ad/commands/info.py +++ b/src/adata/commands/info.py @@ -1,12 +1,11 @@ from pathlib import Path from typing import Any, Optional -import rich from rich.console import Console from rich.tree import Tree -from h5ad.core.info import axis_len, format_type_info, get_entry_type -from h5ad.storage import is_dataset, is_group, open_store +from adata.core.info import axis_len, format_type_info, get_entry_type +from adata.storage import is_dataset, is_group, open_store # Preferred display order for top-level keys KEY_ORDER = ["X", "obs", "var", "obsm", "varm", "layers", "obsp", "varp", "uns"] @@ -24,6 +23,7 @@ def show_info( show_types: bool = False, depth: Optional[int] = None, entry_path: Optional[str] = None, + out_console: Optional[Console] = None, ) -> None: """ Show high-level information about the .h5ad file. @@ -33,23 +33,28 @@ def show_info( show_types (bool): Show detailed type information for each entry depth (Optional[int]): Maximum recursion depth for type display (only with show_types=True) entry_path (Optional[str]): Specific entry path to inspect (e.g., 'obsm/X_pca') + out_console (Optional[Console]): Where results go. Defaults to stdout so + that output pipes cleanly; `console` keeps status and errors. """ + out = out_console if out_console is not None else Console() with open_store(file, "r") as store: f = store.root # If a specific path is requested, show detailed info for that object if entry_path: - _show_object_info(f, entry_path, console) + _show_object_info(f, entry_path, out) return # Get n_obs and n_var n_obs = axis_len(f, "obs") n_var = axis_len(f, "var") - rich.print( - f"[bold cyan]An object with n_obs × n_var: {n_obs if n_obs is not None else '?'} × {n_var if n_var is not None else '?'}[/]" + out.print( + "[bold cyan]An object with n_obs × n_var: " + f"{n_obs if n_obs is not None else '?'} × " + f"{n_var if n_var is not None else '?'}[/]" ) if show_types: - _show_types_tree(f, console, root_label=str(file), depth=depth) + _show_types_tree(f, out, root_label=str(file), depth=depth) else: # List top-level keys and their sub-keys (original behavior) for key in _sort_keys(list(f.keys())): @@ -62,7 +67,7 @@ def show_info( if k not in ("_index", "__categories", "obs_names", "var_names") ] if sub_keys and key != "X": - rich.print( + out.print( f"\t[bold yellow]{key}:[/]\t" + ", ".join(f"[bright_white]{sub}[/]" for sub in sub_keys) ) diff --git a/src/adata/commands/ls.py b/src/adata/commands/ls.py new file mode 100644 index 0000000..1c04003 --- /dev/null +++ b/src/adata/commands/ls.py @@ -0,0 +1,114 @@ +"""`ls` -- list the contents of any HDF5 or Zarr store. + +Unlike `view`, this makes no AnnData assumptions: there is no n_obs x n_var +header and no requirement that `obs`/`var` exist, so it works on `.loom` files +and arbitrary `.h5` stores as well as AnnData ones. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any, Iterator, Optional, Tuple + +from rich.console import Console +from rich.tree import Tree + +from adata.core.info import format_type_info, get_entry_type +from adata.storage import is_dataset, is_group, open_store +from adata.util.path import norm_path + + +def walk(obj: Any, prefix: str = "", depth: Optional[int] = None) -> Iterator[Tuple[str, Any]]: + """Yield ``(path, element)`` for every member below `obj`, depth-first.""" + if not is_group(obj): + return + for key in sorted(obj.keys()): + child = obj[key] + path = f"{prefix}/{key}" if prefix else key + yield path, child + if is_group(child) and (depth is None or depth > 1): + next_depth = None if depth is None else depth - 1 + yield from walk(child, path, next_depth) + + +def _describe(obj: Any, long: bool) -> str: + """Render the trailing annotation for one entry.""" + if not long: + return "" + info = get_entry_type(obj) + bits = [format_type_info(info)] + shape = getattr(obj, "shape", None) + if shape: + bits.append(f"[dim]{tuple(shape)}[/]") + if is_dataset(obj) and getattr(obj, "dtype", None) is not None: + bits.append(f"[dim]{obj.dtype}[/]") + if info["encoding"]: + bits.append(f"[dim]{info['encoding']}[/]") + return " " + " ".join(bits) + + +def list_store( + file: Path, + console: Console, + entry_path: Optional[str] = None, + depth: Optional[int] = None, + long: bool = False, + plain: bool = False, +) -> None: + """Print the structure of a store, as a tree or as bare paths. + + `plain` emits one path per line with no markup, so the output pipes + cleanly into grep or xargs. + """ + with open_store(file, "r", require_anndata=False) as store: + root = store.root + if entry_path: + entry_path = norm_path(entry_path) + if entry_path not in root: + raise KeyError(f"'{entry_path}' not found in the store.") + root = root[entry_path] + + if plain: + _print_plain(root, entry_path or "", depth) + return + + _print_tree(root, console, str(file), entry_path, depth, long) + + +def _print_plain(root: Any, prefix: str, depth: Optional[int]) -> None: + if is_dataset(root): + sys.stdout.write(f"{prefix}\n") + return + for path, _ in walk(root, prefix, depth): + sys.stdout.write(f"{path}\n") + + +def _print_tree( + root: Any, + console: Console, + label: str, + entry_path: Optional[str], + depth: Optional[int], + long: bool, +) -> None: + root_label = f"{label}:{entry_path}" if entry_path else label + tree = Tree(f"[bold]{root_label}[/]") + + if is_dataset(root): + tree.add(f"[bright_white]{entry_path or label}[/]{_describe(root, long)}") + console.print(tree) + return + + nodes = {"": tree} + for path, obj in walk(root, "", depth): + parent_path, _, name = path.rpartition("/") + parent = nodes.get(parent_path, tree) + if is_group(obj): + nodes[path] = parent.add( + f"[bold yellow]{name}/[/]{_describe(obj, long)}" + ) + else: + parent.add(f"[bright_white]{name}[/]{_describe(obj, long)}") + + console.print(tree) diff --git a/src/h5ad/commands/subset.py b/src/adata/commands/subset.py similarity index 90% rename from src/h5ad/commands/subset.py rename to src/adata/commands/subset.py index 940ef07..bd3ddfa 100644 --- a/src/h5ad/commands/subset.py +++ b/src/adata/commands/subset.py @@ -1,4 +1,4 @@ -from h5ad.core.subset import ( +from adata.core.subset import ( _read_name_file, indices_from_name_set, subset_axis_group, diff --git a/src/h5ad/core/__init__.py b/src/adata/core/__init__.py similarity index 100% rename from src/h5ad/core/__init__.py rename to src/adata/core/__init__.py diff --git a/src/adata/core/info.py b/src/adata/core/info.py new file mode 100644 index 0000000..6b777ee --- /dev/null +++ b/src/adata/core/info.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +from typing import Optional, Tuple, Dict, Any, Union + +import numpy as np + +from adata.elements import spec +from adata.elements.read import dataframe_columns, element_len, resolve_index +from adata.storage import is_dataset, is_group, is_hdf5_dataset + + +def _decode_attr(value: Any) -> Any: + if isinstance(value, bytes): + return value.decode("utf-8") + return value + + +def _base_result() -> Dict[str, Any]: + return { + "type": "unknown", + "export_as": None, + "encoding": None, + "shape": None, + "dtype": None, + "details": "", + "version": None, + "ordered": None, + } + + +#: Declared encoding-type -> (reported type, suggested export format). +_ENCODING_TYPES = { + spec.ANNDATA: ("anndata", None), + spec.RAW: ("raw", None), + spec.DICT: ("dict", "json"), + spec.DATAFRAME: ("dataframe", "csv"), + spec.ARRAY: ("array", "npy"), + spec.STRING_ARRAY: ("string-array", "csv"), + spec.CATEGORICAL: ("categorical", "csv"), + spec.CSR_MATRIX: ("sparse-matrix", "mtx"), + spec.CSC_MATRIX: ("sparse-matrix", "mtx"), + spec.NULLABLE_INTEGER: ("nullable-array", "csv"), + spec.NULLABLE_BOOLEAN: ("nullable-array", "csv"), + spec.NULLABLE_STRING_ARRAY: ("nullable-array", "csv"), + spec.NUMERIC_SCALAR: ("scalar", "json"), + spec.STRING: ("scalar", "json"), + spec.NULL: ("null", "json"), + spec.AWKWARD_ARRAY: ("awkward-array", "json"), +} + + +def get_entry_type(entry: Any) -> Dict[str, Any]: + """Describe an element: its type, shape, and how it can be exported. + + The declared `encoding-type` is authoritative and is consulted first; + structural inspection is only a fallback, for files written by anndata + 0.7.x and earlier which carry no encoding attributes. Inferring first + would let a group merely *containing* a member called `obs_names` be + misreported as a dataframe. + """ + result = _base_result() + + enc, enc_ver = spec.encoding_of(entry) + result["encoding"] = enc + result["version"] = enc_ver + + if is_dataset(entry): + result["shape"] = entry.shape + result["dtype"] = str(entry.dtype) + + if enc in _ENCODING_TYPES: + result["type"], result["export_as"] = _ENCODING_TYPES[enc] + _describe(entry, result, enc) + return result + + return _infer_untagged(entry, result) + + +def _describe(entry: Any, result: Dict[str, Any], enc: str) -> None: + """Fill in the human-readable detail line for a tagged element.""" + if enc in spec.SPARSE_TYPES: + shape = entry.attrs.get("shape", None) + shape_str = f"{int(shape[0])}x{int(shape[1])}" if shape is not None else "?" + kind = enc.replace("_matrix", "").upper() + result["details"] = f"Sparse {kind} matrix {shape_str}" + return + + if enc == spec.CATEGORICAL: + result["ordered"] = bool(spec.decode_attr(entry.attrs.get("ordered", False))) + n_codes = _safe_len(entry.get("codes")) + n_cats = _safe_len(entry.get("categories")) + order = ", ordered" if result["ordered"] else "" + result["details"] = f"Categorical [{n_codes} values, {n_cats} categories{order}]" + return + + if enc == spec.DATAFRAME: + cols = dataframe_columns(entry) + legacy = " (legacy __categories)" if "__categories" in entry else "" + result["details"] = f"DataFrame with {len(cols)} columns{legacy}" + return + + if enc in spec.MASKED_TYPES: + n = _safe_len(entry.get("values")) + na = spec.decode_attr(entry.attrs.get("na-value", None)) + na_str = f", na-value={na}" if na is not None else "" + result["details"] = f"Nullable array [{n} values] ({enc}{na_str})" + return + + if enc == spec.STRING: + result["details"] = "String scalar" + return + + if enc == spec.NUMERIC_SCALAR: + result["details"] = f"Numeric scalar ({entry.dtype})" + return + + if enc == spec.NULL: + result["details"] = "Null value" + return + + if enc == spec.AWKWARD_ARRAY: + result["details"] = f"Awkward array (length={entry.attrs.get('length', '?')})" + return + + if enc == spec.STRING_ARRAY: + n = entry.shape[0] if getattr(entry, "shape", None) else "?" + result["details"] = f"String array [{n}]" + return + + if enc in spec.MAPPING_TYPES: + result["details"] = f"Group with {len(list(entry.keys()))} keys" + return + + if enc == spec.ARRAY: + result["details"] = _array_details(entry) + + +def _safe_len(obj: Any) -> Any: + if obj is None: + return "?" + shape = getattr(obj, "shape", None) + return shape[0] if shape else "?" + + +def _array_details(entry: Any) -> str: + if entry.shape == (): + return f"Scalar value ({entry.dtype})" + if entry.ndim == 1: + return f"1D array [{entry.shape[0]}] ({entry.dtype})" + if entry.ndim == 2: + return f"Dense matrix {entry.shape[0]}x{entry.shape[1]} ({entry.dtype})" + return f"{entry.ndim}D array {entry.shape} ({entry.dtype})" + + +def _infer_untagged(entry: Any, result: Dict[str, Any]) -> Dict[str, Any]: + """Classify an element that carries no encoding attributes (anndata <= 0.7).""" + if is_dataset(entry): + if "categories" in entry.attrs: + result["type"] = "categorical" + result["export_as"] = "csv" + result["version"] = result["version"] or "0.1.0" + n_cats = "?" + if is_hdf5_dataset(entry): + try: + n_cats = entry.file[entry.attrs["categories"]].shape[0] + except Exception: + n_cats = "?" + result["details"] = ( + f"Legacy categorical [{entry.shape[0]} values, {n_cats} categories]" + ) + return result + + if entry.shape == (): + result["type"] = "scalar" + result["export_as"] = "json" + result["details"] = f"Scalar value ({entry.dtype})" + return result + + result["type"] = "dense-matrix" if entry.ndim == 2 else "array" + result["export_as"] = "npy" + result["details"] = _array_details(entry) + return result + + if is_group(entry): + if "codes" in entry and "categories" in entry: + result["type"] = "categorical" + result["export_as"] = "csv" + result["details"] = ( + f"Categorical [{_safe_len(entry.get('codes'))} values, " + f"{_safe_len(entry.get('categories'))} categories]" + ) + return result + + if "values" in entry and "mask" in entry: + result["type"] = "nullable-array" + result["export_as"] = "csv" + result["details"] = f"Nullable array [{_safe_len(entry.get('values'))} values]" + return result + + if "_index" in entry.attrs or "obs_names" in entry or "var_names" in entry: + result["type"] = "dataframe" + result["export_as"] = "csv" + result["version"] = result["version"] or "0.1.0" + legacy = " (legacy v0.1.0)" if "__categories" in entry else "" + result["details"] = ( + f"DataFrame with {len(dataframe_columns(entry))} columns{legacy}" + ) + return result + + result["type"] = "dict" + result["export_as"] = "json" + result["details"] = f"Group with {len(list(entry.keys()))} keys" + return result + + return result + + +def format_type_info(info: Dict[str, Any]) -> str: + type_colors = { + "anndata": "cyan", + "raw": "cyan", + "nullable-array": "blue", + "string-array": "green", + "null": "dim", + "awkward-array": "magenta", + "dataframe": "green", + "sparse-matrix": "magenta", + "dense-matrix": "blue", + "array": "blue", + "dict": "yellow", + "categorical": "green", + "scalar": "white", + "unknown": "red", + } + + color = type_colors.get(info["type"], "white") + return f"[{color}]<{info['type']}>[/]" + + +def axis_len(file: Any, axis: str) -> int: + """Number of rows along `axis` ("obs" or "var"). + + Resolves the index through :func:`resolve_index`, so this works whether the + index is a plain dataset or -- as anndata >= 0.11 writes it -- a + `nullable-string-array` group of `values` and `mask`. + """ + if axis not in file: + raise KeyError(f"'{axis}' not found in the file.") + + group = file[axis] + if not is_group(group): + raise TypeError(f"'{axis}' is not a group.") + + if axis not in ("obs", "var"): + raise ValueError(f"Invalid axis '{axis}'. Must be 'obs' or 'var'.") + + index, index_name = resolve_index(group, axis) + try: + return element_len(index) + except (TypeError, ValueError) as exc: + raise ValueError( + f"Cannot determine length of '{axis}': index {index_name!r} " + f"has no usable length ({exc})." + ) from exc + + +def get_axis_group(file: Any, axis: str) -> Tuple[Any, int, str]: + """Return the ``(group, length, index_name)`` triple for an axis.""" + if axis not in ("obs", "var"): + raise ValueError("axis must be 'obs' or 'var'.") + if axis not in file: + raise KeyError(f"'{axis}' not found in the file.") + + group = file[axis] + _, index_name = resolve_index(group, axis) + return group, axis_len(file, axis), index_name diff --git a/src/adata/core/read.py b/src/adata/core/read.py new file mode 100644 index 0000000..4738efd --- /dev/null +++ b/src/adata/core/read.py @@ -0,0 +1,28 @@ +"""Column reading. The implementation now lives in :mod:`adata.elements.read`. + +Kept as a stable import path for existing callers and tests. +""" + +from __future__ import annotations + +from adata.elements.read import ( + col_chunk_as_strings, + decode_str_array, + element_len, + read_categorical_column, + read_masked_column, + read_str_all, + read_str_chunk, + resolve_index, +) + +__all__ = [ + "col_chunk_as_strings", + "decode_str_array", + "element_len", + "read_categorical_column", + "read_masked_column", + "read_str_all", + "read_str_chunk", + "resolve_index", +] diff --git a/src/h5ad/core/subset.py b/src/adata/core/subset.py similarity index 52% rename from src/h5ad/core/subset.py rename to src/adata/core/subset.py index da66a4b..1dba540 100644 --- a/src/h5ad/core/subset.py +++ b/src/adata/core/subset.py @@ -17,8 +17,15 @@ TimeElapsedColumn, ) -from h5ad.core.read import decode_str_array -from h5ad.storage import ( +from adata.elements import spec +from adata.elements.write import set_shape_attr, write_mapping +from adata.elements.read import ( + decode_str_array, + element_len, + read_str_chunk, + resolve_index, +) +from adata.storage import ( create_dataset, copy_attrs, copy_tree, @@ -37,7 +44,13 @@ def _target_backend(dst_group: Any) -> str: def _ensure_group(parent: Any, name: str) -> Any: - return parent[name] if name in parent else parent.create_group(name) + """Get or create an AnnData mapping group, tagged `encoding-type: dict`. + + Every container here (layers, obsm, obsp, varm, varp) is a mapping in the + spec; leaving one untagged makes anndata fall back to its legacy reader + and emit an OldFormatWarning. + """ + return write_mapping(parent, name) def _group_get(parent: Any, key: str) -> Any | None: @@ -45,6 +58,7 @@ def _group_get(parent: Any, key: str) -> Any | None: def _ensure_optional_anndata_groups(dst: Any) -> None: + """Create the optional mapping groups an AnnData store is expected to have.""" for key in ("layers", "obsm", "obsp", "varm", "varp"): _ensure_group(dst, key) @@ -71,20 +85,22 @@ def indices_from_name_set( *, chunk_size: int = 200_000, ) -> Tuple[np.ndarray, Set[str]]: - if names_ds.ndim != 1: - flat_len = int(np.prod(names_ds.shape)) - else: - flat_len = names_ds.shape[0] + """Resolve a set of names to sorted row indices, streaming the index. + + `names_ds` may be a dataset or -- as anndata >= 0.11 writes indices -- a + `nullable-string-array` group, so it is read through + :func:`adata.elements.read.read_str_chunk` rather than sliced directly. + Returns the indices found and the names that were not. + """ + flat_len = element_len(names_ds) remaining = set(keep) found_indices: List[int] = [] + cache: Dict[str, np.ndarray] = {} for start in range(0, flat_len, chunk_size): end = min(start + chunk_size, flat_len) - chunk = names_ds[start:end] - chunk = decode_str_array(np.asarray(chunk)).astype(str) - - for i, name in enumerate(chunk): + for i, name in enumerate(read_str_chunk(names_ds, start, end, cache)): if name in remaining: found_indices.append(start + i) remaining.remove(name) @@ -95,60 +111,98 @@ def indices_from_name_set( return np.asarray(found_indices, dtype=np.int64), remaining +def _take_rows(obj: Any, indices: Optional[np.ndarray]) -> Any: + """Read the selected rows of a dataset, handling both backends' indexing.""" + if indices is None: + return obj[...] + if is_zarr_array(obj): + if obj.ndim == 1: + return obj.oindex[indices] + return obj.oindex[(indices,) + (slice(None),) * (obj.ndim - 1)] + return obj[indices, ...] + + +def _copy_rows( + src_ds: Any, + dst_parent: Any, + name: str, + indices: Optional[np.ndarray], +) -> Any: + """Write the selected rows of `src_ds` into `dst_parent` as `name`.""" + if indices is None: + return copy_tree(src_ds, dst_parent, name) + + target_backend = _target_backend(dst_parent) + kw = dataset_create_kwargs(src_ds, target_backend=target_backend) + kw = _clamp_chunks(kw, len(indices)) + ds = create_dataset( + dst_parent, + name, + data=_take_rows(src_ds, indices), + **kw, + ) + copy_attrs(src_ds.attrs, ds.attrs, target_backend=target_backend) + return ds + + +def _clamp_chunks(kw: dict, n_rows: int) -> dict: + """Shrink a forwarded chunk shape to fit the subset. + + h5py rejects a chunk larger than the dataset, so a chunked source column + subset below its own chunk size would otherwise fail outright. + """ + chunks = kw.get("chunks") + if isinstance(chunks, (tuple, list)) and len(chunks) >= 1 and n_rows > 0: + clamped = (min(int(chunks[0]), n_rows),) + tuple(int(c) for c in chunks[1:]) + kw = dict(kw) + kw["chunks"] = clamped + return kw + + def subset_axis_group( src: Any, dst: Any, indices: Optional[np.ndarray], ) -> None: - copy_attrs(src.attrs, dst.attrs, target_backend=_target_backend(dst)) + """Copy a dataframe group, taking only `indices` along its rows. + + Every column layout the spec allows has to be narrowed here, not just plain + datasets: `categorical` keeps its categories and subsets only `codes`, + while the masked layouts (`nullable-*`, which anndata >= 0.11 uses for the + index and every string column) subset both `values` and `mask`. Copying a + masked column whole would leave it longer than the rest of the frame. + """ target_backend = _target_backend(dst) + copy_attrs(src.attrs, dst.attrs, target_backend=target_backend) for key in src.keys(): obj = src[key] if is_dataset(obj): - if indices is None: - copy_tree(obj, dst, key) - else: - if is_zarr_array(obj): - if obj.ndim == 1: - data = obj.oindex[indices] - else: - selection = (indices,) + (slice(None),) * (obj.ndim - 1) - data = obj.oindex[selection] - else: - data = obj[indices, ...] - ds = create_dataset( - dst, - key, - data=data, - **dataset_create_kwargs(obj, target_backend=target_backend), - ) - copy_attrs(obj.attrs, ds.attrs, target_backend=target_backend) - elif is_group(obj): - enc = obj.attrs.get("encoding-type", b"") - if isinstance(enc, bytes): - enc = enc.decode("utf-8") - - if enc == "categorical": - gdst = dst.create_group(key) - copy_attrs(obj.attrs, gdst.attrs, target_backend=target_backend) - copy_tree(obj["categories"], gdst, "categories") - - codes = obj["codes"] - if indices is None: - copy_tree(codes, gdst, "codes") - else: - codes_sub = codes[indices, ...] - ds = create_dataset( - gdst, - "codes", - data=codes_sub, - **dataset_create_kwargs(codes, target_backend=target_backend), - ) - copy_attrs(codes.attrs, ds.attrs, target_backend=target_backend) - else: - copy_tree(obj, dst, key) + _copy_rows(obj, dst, key, indices) + continue + + if not is_group(obj): + continue + + enc = _decode_attr(obj.attrs.get("encoding-type", b"")) + + if enc == spec.CATEGORICAL or (enc is None and "codes" in obj): + gdst = dst.create_group(key) + copy_attrs(obj.attrs, gdst.attrs, target_backend=target_backend) + copy_tree(obj["categories"], gdst, "categories") + _copy_rows(obj["codes"], gdst, "codes", indices) + continue + + if enc in spec.MASKED_TYPES or ("values" in obj and "mask" in obj): + gdst = dst.create_group(key) + copy_attrs(obj.attrs, gdst.attrs, target_backend=target_backend) + _copy_rows(obj["values"], gdst, "values", indices) + _copy_rows(obj["mask"], gdst, "mask", indices) + continue + + # Not row-aligned (e.g. __categories) -- copy verbatim. + copy_tree(obj, dst, key) def subset_dense_matrix( @@ -198,96 +252,122 @@ def subset_dense_matrix( dst[out_start:out_end, :] = block +def _minor_remap(keep: Optional[np.ndarray], size: int) -> Optional[np.ndarray]: + """Build a lookup from old minor index to new, with -1 for dropped entries. + + A dense lookup table costs one int32 per column of the source, which is + negligible beside the matrix itself and turns the remap into a single + vectorised gather rather than a per-entry dict lookup. + """ + if keep is None: + return None + remap = np.full(size, -1, dtype=np.int64) + remap[keep] = np.arange(len(keep), dtype=np.int64) + return remap + + def subset_sparse_matrix_group( src: Any, dst_parent: Any, name: str, obs_idx: Optional[np.ndarray], var_idx: Optional[np.ndarray], + *, + chunk_major: int = 4096, ) -> None: - enc = src.attrs.get("encoding-type", b"") - if isinstance(enc, bytes): - enc = enc.decode("utf-8") - - if enc not in ("csr_matrix", "csc_matrix"): + """Subset a CSR/CSC matrix, streaming it a block of major axis at a time. + + Only the slice of `data`/`indices` spanned by the current block is read, so + peak memory is set by `chunk_major` rather than by the matrix. The output + datasets are grown as each block is appended, since the final nnz is not + known until the pass completes. + """ + enc = _decode_attr(src.attrs.get("encoding-type", b"")) + if enc not in spec.SPARSE_TYPES: raise ValueError(f"Unsupported sparse encoding type: {enc}") - data = np.asarray(src["data"][...]) - indices = np.asarray(src["indices"][...], dtype=np.int64) - indptr = np.asarray(src["indptr"][...], dtype=np.int64) shape = src.attrs.get("shape", None) if shape is None: raise ValueError("Sparse matrix group missing 'shape' attribute.") n_rows, n_cols = int(shape[0]), int(shape[1]) - if enc == "csr_matrix": - row_idx = obs_idx if obs_idx is not None else np.arange(n_rows, dtype=np.int64) - col_idx = var_idx if var_idx is not None else np.arange(n_cols, dtype=np.int64) - - new_data = [] - new_indices = [] - new_indptr = [0] - - for r in row_idx: - start = indptr[r] - end = indptr[r + 1] - row_cols = indices[start:end] - row_data = data[start:end] - - if var_idx is not None: - col_mask = np.isin(row_cols, col_idx) - row_cols = row_cols[col_mask] - row_data = row_data[col_mask] - - if var_idx is not None: - col_map = {c: i for i, c in enumerate(col_idx)} - row_cols = np.array([col_map[c] for c in row_cols], dtype=np.int64) - - new_indices.extend(row_cols.tolist()) - new_data.extend(row_data.tolist()) - new_indptr.append(len(new_indices)) + data_ds, indices_ds = src["data"], src["indices"] + indptr = np.asarray(src["indptr"][...], dtype=np.int64) - new_shape = (len(row_idx), len(col_idx)) + if enc == spec.CSR_MATRIX: + major_idx, minor_keep, n_minor = obs_idx, var_idx, n_cols + out_rows = len(obs_idx) if obs_idx is not None else n_rows + out_cols = len(var_idx) if var_idx is not None else n_cols else: - row_idx = obs_idx if obs_idx is not None else np.arange(n_rows, dtype=np.int64) - col_idx = var_idx if var_idx is not None else np.arange(n_cols, dtype=np.int64) - - new_data = [] - new_indices = [] - new_indptr = [0] - - for c in col_idx: - start = indptr[c] - end = indptr[c + 1] - col_rows = indices[start:end] - col_data = data[start:end] + major_idx, minor_keep, n_minor = var_idx, obs_idx, n_rows + out_rows = len(obs_idx) if obs_idx is not None else n_rows + out_cols = len(var_idx) if var_idx is not None else n_cols - if obs_idx is not None: - row_mask = np.isin(col_rows, row_idx) - col_rows = col_rows[row_mask] - col_data = col_data[row_mask] + n_major = n_rows if enc == spec.CSR_MATRIX else n_cols + majors = major_idx if major_idx is not None else np.arange(n_major, dtype=np.int64) + remap = _minor_remap(minor_keep, n_minor) - if obs_idx is not None: - row_map = {r: i for i, r in enumerate(row_idx)} - col_rows = np.array([row_map[r] for r in col_rows], dtype=np.int64) - - new_indices.extend(col_rows.tolist()) - new_data.extend(col_data.tolist()) - new_indptr.append(len(new_indices)) + group = dst_parent.create_group(name) + copy_attrs(src.attrs, group.attrs, target_backend=_target_backend(dst_parent)) + spec.set_encoding(group, enc) + set_shape_attr(group, (out_rows, out_cols)) + + out_data = _growable(group, "data", data_ds.dtype) + out_indices = _growable(group, "indices", np.int64) + out_indptr = [0] + nnz = 0 + + for block_start in range(0, len(majors), chunk_major): + block = majors[block_start : block_start + chunk_major] + # One contiguous read covers the whole block's entries. + lo, hi = int(indptr[block].min()), int(indptr[block + 1].max()) + if hi > lo: + block_indices = np.asarray(indices_ds[lo:hi], dtype=np.int64) + block_data = np.asarray(data_ds[lo:hi]) + else: + block_indices = np.empty(0, dtype=np.int64) + block_data = np.empty(0, dtype=data_ds.dtype) + + kept_indices: List[np.ndarray] = [] + kept_data: List[np.ndarray] = [] + for m in block: + sl = slice(int(indptr[m]) - lo, int(indptr[m + 1]) - lo) + minor = block_indices[sl] + values = block_data[sl] + if remap is not None: + mapped = remap[minor] + keep = mapped >= 0 + minor, values = mapped[keep], values[keep] + kept_indices.append(minor) + kept_data.append(values) + nnz += len(minor) + out_indptr.append(nnz) + + if kept_indices: + _append(out_indices, np.concatenate(kept_indices)) + _append(out_data, np.concatenate(kept_data)) + + create_dataset( + group, "indptr", data=np.asarray(out_indptr, dtype=np.int64) + ) - new_shape = (len(row_idx), len(col_idx)) - group = dst_parent.create_group(name) - group.attrs["encoding-type"] = enc - group.attrs["encoding-version"] = "0.1.0" +def _growable(group: Any, name: str, dtype: Any) -> Any: + """Create an empty 1-D dataset that can be extended as blocks arrive.""" if is_zarr_group(group): - group.attrs["shape"] = list(new_shape) - else: - group.attrs["shape"] = np.array(new_shape, dtype=np.int64) + return group.create_array(name, shape=(0,), dtype=dtype, chunks=(65536,)) + return group.create_dataset( + name, shape=(0,), maxshape=(None,), dtype=dtype, chunks=(65536,) + ) - create_dataset(group, "data", data=np.array(new_data, dtype=data.dtype)) - create_dataset(group, "indices", data=np.array(new_indices, dtype=indices.dtype)) - create_dataset(group, "indptr", data=np.array(new_indptr, dtype=indptr.dtype)) + +def _append(ds: Any, values: np.ndarray) -> None: + """Append a block to a growable 1-D dataset.""" + if values.size == 0: + return + start = ds.shape[0] + ds.resize((start + values.size,)) + ds[start:] = values def subset_matrix_entry( @@ -307,17 +387,88 @@ def subset_matrix_entry( return if is_group(obj): - enc = obj.attrs.get("encoding-type", b"") - if isinstance(enc, bytes): - enc = enc.decode("utf-8") - if enc in ("csr_matrix", "csc_matrix"): + enc = _decode_attr(obj.attrs.get("encoding-type", b"")) + if enc in spec.SPARSE_TYPES: subset_sparse_matrix_group(obj, dst_parent, name, obs_idx, var_idx) return + if enc == spec.DATAFRAME: + # obsm/varm may hold a dataframe; it is row-aligned like obs/var. + subset_axis_group(obj, dst_parent.create_group(name), obs_idx) + return raise ValueError(f"Unsupported {entry_label} encoding type: {enc}") raise ValueError(f"Unsupported {entry_label} object type") +HANDLED_KEYS = frozenset( + {"obs", "var", "X", "layers", "obsm", "varm", "obsp", "varp", "uns", "raw"} +) + + +def subset_raw_group( + src_raw: Any, + dst: Any, + obs_idx: Optional[np.ndarray], + var_keep: Optional[Set[str]], + *, + chunk_rows: int, + console: Console, +) -> None: + """Subset a `raw/` group, which carries its own var axis. + + `raw` typically holds more genes than the main object, so its var names are + matched independently rather than reusing the outer var indices -- using + those would select the wrong columns entirely. + """ + raw_dst = dst.create_group("raw") + copy_attrs(src_raw.attrs, raw_dst.attrs, target_backend=_target_backend(dst)) + spec.set_encoding(raw_dst, spec.RAW) + + raw_var_idx: Optional[np.ndarray] = None + if var_keep is not None and "var" in src_raw: + raw_var_names, _ = resolve_index(src_raw["var"], "var") + raw_var_idx, missing = indices_from_name_set(raw_var_names, var_keep) + console.print( + f"[green]Selected {len(raw_var_idx)} raw/var " + f"(of {element_len(raw_var_names)})[/]" + ) + if missing: + console.print( + f"[yellow]Warning: {len(missing)} var names not found in raw/var[/]" + ) + + if "var" in src_raw: + subset_axis_group(src_raw["var"], raw_dst.create_group("var"), raw_var_idx) + + if "X" in src_raw: + subset_matrix_entry( + src_raw["X"], + raw_dst, + "X", + obs_idx, + raw_var_idx, + chunk_rows=chunk_rows, + entry_label="raw/X", + ) + + if "varm" in src_raw: + varm_dst = _ensure_group(raw_dst, "varm") + for key in src_raw["varm"].keys(): + subset_matrix_entry( + src_raw["varm"][key], + varm_dst, + key, + raw_var_idx, + None, + chunk_rows=chunk_rows, + entry_label=f"raw/varm:{key}", + ) + + for key in src_raw.keys(): + if key not in ("X", "var", "varm"): + copy_tree(src_raw[key], raw_dst, key) + + def subset_h5ad( file: Path, output: Optional[Path], @@ -366,12 +517,7 @@ def subset_h5ad( if obs_keep is not None: console.print("[cyan]Matching obs names...[/]") obs_group = src["obs"] - obs_index = _decode_attr(obs_group.attrs.get("_index", "obs_names")) - obs_names_ds = _group_get(obs_group, "obs_names") or _group_get( - obs_group, obs_index - ) - if obs_names_ds is None: - raise KeyError("Could not find obs names") + obs_names_ds, _ = resolve_index(obs_group, "obs") obs_idx, missing_obs = indices_from_name_set(obs_names_ds, obs_keep) if missing_obs: @@ -379,19 +525,14 @@ def subset_h5ad( f"[yellow]Warning: {len(missing_obs)} obs names not found in file[/]" ) console.print( - f"[green]Selected {len(obs_idx)} obs (of {obs_names_ds.shape[0]})[/]" + f"[green]Selected {len(obs_idx)} obs (of {element_len(obs_names_ds)})[/]" ) var_idx = None if var_keep is not None: console.print("[cyan]Matching var names...[/]") var_group = src["var"] - var_index = _decode_attr(var_group.attrs.get("_index", "var_names")) - var_names_ds = _group_get(var_group, "var_names") or _group_get( - var_group, var_index - ) - if var_names_ds is None: - raise KeyError("Could not find var names") + var_names_ds, _ = resolve_index(var_group, "var") var_idx, missing_var = indices_from_name_set(var_names_ds, var_keep) if missing_var: @@ -399,7 +540,7 @@ def subset_h5ad( f"[yellow]Warning: {len(missing_var)} var names not found in file[/]" ) console.print( - f"[green]Selected {len(var_idx)} var (of {var_names_ds.shape[0]})[/]" + f"[green]Selected {len(var_idx)} var (of {element_len(var_names_ds)})[/]" ) tasks: List[str] = [] @@ -421,6 +562,17 @@ def subset_h5ad( tasks.extend([f"varp:{k}" for k in src["varp"].keys()]) if "uns" in src: tasks.append("uns") + if "raw" in src: + tasks.append("raw") + + passthrough = [k for k in src.keys() if k not in HANDLED_KEYS] + if passthrough: + console.print( + "[yellow]Copying unrecognised top-level " + f"{'keys' if len(passthrough) > 1 else 'key'} verbatim: " + f"{', '.join(sorted(passthrough))}[/]" + ) + tasks.extend(f"copy:{k}" for k in passthrough) with Progress( SpinnerColumn(finished_text="[green]✓[/]"), @@ -515,6 +667,18 @@ def subset_h5ad( ) elif task == "uns": copy_tree(src["uns"], dst, "uns") + elif task == "raw": + subset_raw_group( + src["raw"], + dst, + obs_idx, + var_keep, + chunk_rows=chunk_rows, + console=console, + ) + elif task.startswith("copy:"): + key = task.split(":", 1)[1] + copy_tree(src[key], dst, key) progress.update( task_id, description=f"[green]Subsetting {task}[/]", diff --git a/src/adata/elements/__init__.py b/src/adata/elements/__init__.py new file mode 100644 index 0000000..138687b --- /dev/null +++ b/src/adata/elements/__init__.py @@ -0,0 +1,10 @@ +"""AnnData on-disk element handling: the spec, and reading/writing each element. + +`spec` holds the encoding constants, `read` turns any on-disk layout into +usable values, `write` emits the current spec, and `strings` reconciles the two +backends' incompatible spellings of text. +""" + +from adata.elements import read, spec, strings, write + +__all__ = ["read", "spec", "strings", "write"] diff --git a/src/adata/elements/read.py b/src/adata/elements/read.py new file mode 100644 index 0000000..f2c20b5 --- /dev/null +++ b/src/adata/elements/read.py @@ -0,0 +1,310 @@ +"""Reading AnnData elements, in every layout the spec has ever used. + +Elements that logically hold one column of text are stored four different ways +depending on which anndata wrote the file: + +* a plain dataset of bytes or vlen str (`string-array`, or untagged in 0.7.x), +* a group of `categories` + `codes` (`categorical`), +* a dataset of codes plus a `categories` attribute (legacy 0.7.x categorical), +* a group of `values` + `mask` (`nullable-string-array`, what anndata >= 0.11 + writes for every string column *including the dataframe index*). + +The last case is why this module exists: code that assumed an index was always +a dataset fails outright on any recent file. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np + +from adata.elements import spec +from adata.storage import is_dataset, is_group, is_hdf5_dataset + + +def decode_str_array(array: np.ndarray) -> np.ndarray: + """Decode an array of bytes/objects to a numpy array of str.""" + arr = np.asarray(array) + + if np.issubdtype(arr.dtype, np.bytes_): + flat = arr.reshape(-1) + decoded = [ + ( + b.decode("utf-8", errors="replace") + if isinstance(b, (bytes, np.bytes_)) + else str(b) + ) + for b in flat + ] + return np.asarray(decoded, dtype=str).reshape(arr.shape) + + # "T" is numpy's StringDType, which zarr-python 3 reports for both + # VariableLengthUTF8 (v3) and VLenUTF8 (v2); it refuses a direct astype(str). + if arr.dtype.kind in ("O", "T"): + flat = arr.reshape(-1) + decoded = [ + ( + v.decode("utf-8", errors="replace") + if isinstance(v, (bytes, np.bytes_)) + else str(v) + ) + for v in flat + ] + return np.asarray(decoded, dtype=str).reshape(arr.shape) + + return arr.astype(str) + + +def element_len(obj: Any) -> int: + """Length along the first axis, whether `obj` is a dataset or a group. + + Handles the masked (`values`/`mask`) and categorical group layouts, so + callers never need to know how a column happens to be stored. + """ + if is_dataset(obj): + shape = getattr(obj, "shape", None) + if not shape: + raise ValueError("Element is a scalar and has no length.") + return int(shape[0]) + + if is_group(obj): + enc = spec.encoding_type(obj) + if enc in spec.MASKED_TYPES or "values" in obj: + return int(obj["values"].shape[0]) + if enc == spec.CATEGORICAL or "codes" in obj: + return int(obj["codes"].shape[0]) + raise TypeError( + f"Cannot determine length of group with encoding {enc!r}: " + "expected 'values' or 'codes'." + ) + + raise TypeError(f"Unsupported element type: {type(obj)}") + + +def resolve_index(group: Any, axis: Optional[str] = None) -> Tuple[Any, str]: + """Return the ``(index_element, index_name)`` of a dataframe group. + + Honours the declared ``_index`` attribute first and only then falls back to + the ``obs_names``/``var_names`` convention -- the reverse of that order + picks the wrong column on a file carrying both. + """ + index_name = spec.decode_attr(group.attrs.get("_index", None)) + + candidates: List[str] = [] + if index_name: + candidates.append(str(index_name)) + if axis == "obs": + candidates.append("obs_names") + elif axis == "var": + candidates.append("var_names") + else: + candidates.extend(("obs_names", "var_names")) + candidates.append("_index") + + for name in candidates: + if name in group: + return group[name], name + + raise KeyError( + f"Could not find an index in group {getattr(group, 'name', '?')!r}; " + f"tried {candidates}." + ) + + +def dataframe_columns(group: Any, index_name: Optional[str] = None) -> List[str]: + """Return a dataframe group's data columns in their authored order. + + The spec records original column order in the `column-order` attribute; + without it the order would be whatever the backend enumerates, which is + alphabetical on HDF5. Columns present on disk but absent from the + attribute are appended so nothing is ever dropped. + """ + if index_name is None: + try: + _, index_name = resolve_index(group) + except KeyError: + index_name = None + + skip = set(spec.RESERVED_DATAFRAME_KEYS) + if index_name: + skip.add(index_name) + + present = [k for k in group.keys() if k not in skip] + + declared = group.attrs.get("column-order", None) + if declared is None: + return present + + ordered = [ + str(spec.decode_attr(c)) + for c in np.asarray(declared).reshape(-1).tolist() + ] + seen = set() + out = [c for c in ordered if c in present and not (c in seen or seen.add(c))] + out.extend(c for c in present if c not in seen) + return out + + +def _categorical_cache_key(col: Any, parent_group: Any | None = None) -> str: + col_name = getattr(col, "name", None) + if isinstance(col_name, str) and col_name: + return col_name + + if parent_group is not None: + parent_name = getattr(parent_group, "name", "") + rel_name = getattr(col, "path", "") + if parent_name or rel_name: + return f"{parent_name}/{rel_name}" + + return repr(col) + + +def read_categories(col: Any, parent_group: Any | None = None) -> np.ndarray: + """Read the category labels of a categorical column, modern or legacy.""" + if is_group(col): + return np.asarray(decode_str_array(col["categories"][...]), dtype=str) + + cats_ref = col.attrs.get("categories", None) + if cats_ref is not None and is_hdf5_dataset(col): + # Legacy 0.7.x HDF5: an object reference into __categories. + return np.asarray(decode_str_array(col.file[cats_ref][...]), dtype=str) + + if parent_group is not None and "__categories" in parent_group: + col_name = str(getattr(col, "name", "")).split("/")[-1] + cats_grp = parent_group["__categories"] + if col_name in cats_grp: + return np.asarray(decode_str_array(cats_grp[col_name][...]), dtype=str) + + if cats_ref is not None and isinstance(cats_ref, (str, bytes)): + # Legacy Zarr: `categories` is an absolute path within the store. + path = spec.decode_attr(cats_ref).lstrip("/") + root = getattr(col, "store_path", None) + if parent_group is not None and path.split("/")[-1] in parent_group: + return np.asarray( + decode_str_array(parent_group[path.split("/")[-1]][...]), dtype=str + ) + del root + + raise KeyError( + f"Cannot find categories for categorical column " + f"{getattr(col, 'name', '?')!r}." + ) + + +def is_ordered(col: Any) -> bool: + """Whether a categorical column declares an order over its categories.""" + return bool(spec.decode_attr(col.attrs.get("ordered", False))) + + +def read_categorical_column( + col: Any, + start: int, + end: int, + cache: Dict[str, np.ndarray], + parent_group: Any | None = None, +) -> List[str]: + """Read rows [start, end) of a categorical column as strings. + + Codes outside the category range -- notably -1 -- render as empty, which is + how the spec denotes a missing value. + """ + key = _categorical_cache_key(col, parent_group) + if key not in cache: + cache[key] = read_categories(col, parent_group) + cats = cache[key] + + codes_ds = col["codes"] if is_group(col) else col + codes = np.asarray(codes_ds[start:end], dtype=np.int64) + return [cats[c] if 0 <= c < len(cats) else "" for c in codes] + + +def _format_values(values: np.ndarray) -> List[str]: + """Render a numeric or text array as strings without a lossy detour.""" + arr = np.asarray(values) + if arr.dtype.kind in ("O", "S", "U", "T"): + return decode_str_array(arr).tolist() + return [str(v) for v in arr.tolist()] + + +def read_masked_column( + col: Any, start: int, end: int, na_repr: str = "" +) -> List[str]: + """Read rows [start, end) of a `values`/`mask` group as strings. + + A True mask marks a missing entry, rendered as `na_repr`. + """ + values = col["values"][start:end] + mask = np.asarray(col["mask"][start:end], dtype=bool) + rendered = _format_values(values) + return [na_repr if m else v for v, m in zip(rendered, mask)] + + +def read_str_chunk( + obj: Any, + start: int, + end: int, + cache: Optional[Dict[str, np.ndarray]] = None, + parent_group: Any | None = None, + na_repr: str = "", +) -> List[str]: + """Read rows [start, end) of any column-like element as strings. + + This is the one entry point for turning a column into text, whatever + layout it uses on disk. + """ + if cache is None: + cache = {} + + if is_dataset(obj): + if "categories" in obj.attrs: + return read_categorical_column(obj, start, end, cache, parent_group) + chunk = obj[start:end] + arr = np.asarray(chunk) + if arr.ndim != 1: + arr = arr.reshape(-1) + return _format_values(arr) + + if is_group(obj): + enc = spec.encoding_type(obj) + + if enc == spec.CATEGORICAL or (enc is None and "codes" in obj): + return read_categorical_column(obj, start, end, cache, parent_group) + + if enc in spec.MASKED_TYPES or (enc is None and "values" in obj and "mask" in obj): + return read_masked_column(obj, start, end, na_repr=na_repr) + + raise ValueError( + f"Unsupported group encoding {enc!r} for element " + f"{getattr(obj, 'name', '?')!r}." + ) + + raise TypeError(f"Unsupported element type: {type(obj)}") + + +def read_str_all(obj: Any, chunk_size: int = 200_000, **kwargs: Any) -> List[str]: + """Read an entire column-like element as strings, in chunks.""" + n = element_len(obj) + cache: Dict[str, np.ndarray] = {} + out: List[str] = [] + for start in range(0, n, chunk_size): + out.extend( + read_str_chunk(obj, start, min(start + chunk_size, n), cache, **kwargs) + ) + return out + + +def col_chunk_as_strings( + group: Any, + col_name: str, + start: int, + end: int, + cat_cache: Dict[str, np.ndarray], +) -> List[str]: + """Read rows [start, end) of a named column within a dataframe group.""" + if col_name not in group: + raise RuntimeError( + f"Column {col_name!r} not found in group " + f"{getattr(group, 'name', '?')!r}" + ) + return read_str_chunk(group[col_name], start, end, cat_cache, parent_group=group) diff --git a/src/adata/elements/spec.py b/src/adata/elements/spec.py new file mode 100644 index 0000000..541dd1a --- /dev/null +++ b/src/adata/elements/spec.py @@ -0,0 +1,107 @@ +"""AnnData on-disk encoding constants and attribute access. + +This is the single source of truth for `encoding-type` / `encoding-version` +pairs. Every read that inspects an encoding and every write that stamps one +goes through here, so the spec version the tool targets is stated in exactly +one place. + +See docs/ELEMENTS_h5ad.md and docs/ELEMENTS_zarr.md for the full spec. +""" + +from __future__ import annotations + +from typing import Any, Optional, Tuple + +ANNDATA = "anndata" +RAW = "raw" +DICT = "dict" +DATAFRAME = "dataframe" +ARRAY = "array" +STRING_ARRAY = "string-array" +CATEGORICAL = "categorical" +CSR_MATRIX = "csr_matrix" +CSC_MATRIX = "csc_matrix" +NULLABLE_INTEGER = "nullable-integer" +NULLABLE_BOOLEAN = "nullable-boolean" +NULLABLE_STRING_ARRAY = "nullable-string-array" +NUMERIC_SCALAR = "numeric-scalar" +STRING = "string" +AWKWARD_ARRAY = "awkward-array" +NULL = "null" + +#: Encoding type -> the version this tool writes. +CURRENT_VERSION = { + ANNDATA: "0.1.0", + RAW: "0.1.0", + DICT: "0.1.0", + DATAFRAME: "0.2.0", + ARRAY: "0.2.0", + STRING_ARRAY: "0.2.0", + CATEGORICAL: "0.2.0", + CSR_MATRIX: "0.1.0", + CSC_MATRIX: "0.1.0", + NULLABLE_INTEGER: "0.1.0", + NULLABLE_BOOLEAN: "0.1.0", + NULLABLE_STRING_ARRAY: "0.1.0", + NUMERIC_SCALAR: "0.2.0", + STRING: "0.2.0", + AWKWARD_ARRAY: "0.1.0", + NULL: "0.1.0", +} + +SPARSE_TYPES = frozenset({CSR_MATRIX, CSC_MATRIX}) +NULLABLE_TYPES = frozenset( + {NULLABLE_INTEGER, NULLABLE_BOOLEAN, NULLABLE_STRING_ARRAY} +) +#: Encodings stored as a group holding `values` and `mask`. +MASKED_TYPES = NULLABLE_TYPES +#: Encodings that yield text when read as strings. +STRING_TYPES = frozenset({STRING_ARRAY, NULLABLE_STRING_ARRAY, STRING, CATEGORICAL}) +#: Group encodings that are plain mappings of further elements. +MAPPING_TYPES = frozenset({DICT, ANNDATA, RAW}) + +#: Keys that are structural rather than dataframe columns. +RESERVED_DATAFRAME_KEYS = frozenset({"_index", "__categories"}) + + +def decode_attr(value: Any) -> Any: + """Decode a raw attribute value to a plain Python value. + + HDF5 hands back `bytes` for string attributes while Zarr hands back `str`; + numpy scalars appear on both. Normalising here is what lets the rest of the + codebase compare attributes with `==`. + """ + if isinstance(value, bytes): + return value.decode("utf-8") + if hasattr(value, "item") and getattr(value, "shape", None) == (): + return value.item() + return value + + +def encoding_of(obj: Any) -> Tuple[Optional[str], Optional[str]]: + """Return the declared ``(encoding-type, encoding-version)`` of an element. + + Either component is None when the element is untagged, which is the normal + case for files written by anndata 0.7.x and earlier. + """ + attrs = getattr(obj, "attrs", {}) + enc = decode_attr(attrs.get("encoding-type", None)) + ver = decode_attr(attrs.get("encoding-version", None)) + return (enc or None, ver or None) + + +def encoding_type(obj: Any) -> Optional[str]: + """Return just the declared ``encoding-type``, or None if untagged.""" + return encoding_of(obj)[0] + + +def set_encoding(obj: Any, enc_type: str, version: Optional[str] = None) -> None: + """Stamp ``encoding-type``/``encoding-version`` on a group or dataset. + + The version defaults to the current spec version for that type, so callers + never hardcode one. + """ + if version is None: + version = CURRENT_VERSION[enc_type] + obj.attrs["encoding-type"] = enc_type + obj.attrs["encoding-version"] = version diff --git a/src/adata/elements/strings.py b/src/adata/elements/strings.py new file mode 100644 index 0000000..7cafe59 --- /dev/null +++ b/src/adata/elements/strings.py @@ -0,0 +1,90 @@ +"""Backend-neutral handling of string elements. + +The AnnData spec requires `string-array` to be variable-length UTF-8: a vlen +str dtype in HDF5, `VariableLengthUTF8` in Zarr. Neither backend accepts the +other's spelling, and a dtype round-tripped verbatim across backends fails +outright -- an h5py vlen dataset reports `dtype == object`, which Zarr rejects +as ambiguous. Everything that creates or copies a string element therefore +resolves the dtype through this module rather than reusing the source's. +""" + +from __future__ import annotations + +from typing import Any, Optional + +import h5py +import numpy as np + +from adata.storage import is_dataset, is_zarr_array + + +def is_string_dtype(dtype: Any) -> bool: + """True if `dtype` holds text, in any spelling either backend produces. + + Covers h5py vlen-UTF-8 (object), fixed-width bytes ('S'), numpy unicode + ('U'), numpy StringDType ('T', what zarr-python 3 reports for both + `VariableLengthUTF8` and v2 `VLenUTF8`), and bare object arrays. + """ + if dtype is None: + return False + if dtype is str: + return True + try: + if h5py.check_string_dtype(dtype) is not None: + return True + except (TypeError, AttributeError): + pass + kind = getattr(dtype, "kind", None) + return kind in ("O", "S", "U", "T") + + +def is_string_element(obj: Any) -> bool: + """True if `obj` is a dataset/array holding text.""" + return is_dataset(obj) and is_string_dtype(getattr(obj, "dtype", None)) + + +def string_dtype_for(backend: str, zarr_format: Optional[int] = None) -> Any: + """Return the spec-compliant variable-length UTF-8 dtype for `backend`. + + `zarr_format` is accepted for symmetry; zarr-python 3 maps `str` to + `VariableLengthUTF8` under both v2 and v3, so it does not currently affect + the result. + """ + if backend == "zarr": + return str + return h5py.string_dtype(encoding="utf-8") + + +def target_dtype(src_dtype: Any, target_backend: str, zarr_format: Optional[int] = None) -> Any: + """Map a source dtype onto one the target backend can actually create. + + String dtypes are normalised to variable-length UTF-8; everything else is + passed through unchanged. + """ + if is_string_dtype(src_dtype): + return string_dtype_for(target_backend, zarr_format) + return src_dtype + + +def as_str_array(values: Any) -> np.ndarray: + """Coerce a sequence to a 1-D numpy array of Python str, decoding bytes.""" + arr = np.asarray(values, dtype=object) + flat = [ + v.decode("utf-8", errors="replace") if isinstance(v, (bytes, np.bytes_)) else str(v) + for v in arr.reshape(-1) + ] + return np.asarray(flat, dtype=object).reshape(arr.shape) + + +def strip_string_filters(kwargs: dict, target_backend: str, zarr_format: Optional[int]) -> dict: + """Drop codecs that cannot survive the jump to `target_backend`. + + A Zarr v2 string array carries `VLenUTF8` in `filters`; forwarding it to a + v3 array raises `TypeError: Expected an ArrayArrayCodec`. The v3 string + dtype encodes variable length itself, so the filter is simply dropped. + """ + if target_backend != "zarr" or zarr_format != 3: + return kwargs + out = dict(kwargs) + out.pop("filters", None) + return out diff --git a/src/adata/elements/write.py b/src/adata/elements/write.py new file mode 100644 index 0000000..4098967 --- /dev/null +++ b/src/adata/elements/write.py @@ -0,0 +1,246 @@ +"""Writing AnnData elements in the current on-disk spec. + +Every element this tool creates is tagged with its `encoding-type` and +`encoding-version` here, so a store the CLI writes is indistinguishable from +one anndata wrote. Text always goes out as variable-length UTF-8 -- fixed-width +bytes are readable by anndata but are not what `string-array` specifies, and on +Zarr v3 they produce a dtype other libraries cannot read. +""" + +from __future__ import annotations + +from typing import Any, Iterable, Optional, Sequence + +import numpy as np + +from adata.elements import spec +from adata.storage import create_dataset, is_zarr_group + + +def _backend_of(parent: Any) -> str: + return "zarr" if is_zarr_group(parent) else "hdf5" + + +def _replace(parent: Any, name: str) -> None: + """Remove an existing member so it can be rewritten.""" + if name in parent: + del parent[name] + + +def write_mapping(parent: Any, name: str, replace: bool = False) -> Any: + """Create (or fetch) a group tagged as a `dict` mapping.""" + if replace: + _replace(parent, name) + group = parent[name] if name in parent else parent.create_group(name) + spec.set_encoding(group, spec.DICT) + return group + + +def write_string_array( + parent: Any, name: str, values: Iterable[Any], replace: bool = False +) -> Any: + """Write a `string-array`: variable-length UTF-8 on either backend.""" + if replace: + _replace(parent, name) + ds = create_dataset(parent, name, data=np.asarray(list(values), dtype=object)) + spec.set_encoding(ds, spec.STRING_ARRAY) + return ds + + +def write_dense( + parent: Any, name: str, data: Any, replace: bool = False, **kwargs: Any +) -> Any: + """Write a dense numeric `array`.""" + if replace: + _replace(parent, name) + ds = create_dataset(parent, name, data=np.asarray(data), **kwargs) + spec.set_encoding(ds, spec.ARRAY) + return ds + + +def write_scalar(parent: Any, name: str, value: Any, replace: bool = False) -> Any: + """Write a 0-d scalar as `string` or `numeric-scalar`, per its type.""" + if replace: + _replace(parent, name) + + if isinstance(value, (str, bytes, np.str_, np.bytes_)): + text = value.decode("utf-8") if isinstance(value, bytes) else str(value) + ds = create_dataset(parent, name, data=np.asarray(text, dtype=object)) + spec.set_encoding(ds, spec.STRING) + return ds + + arr = np.asarray(value) + ds = create_dataset(parent, name, data=arr) + spec.set_encoding(ds, spec.NUMERIC_SCALAR) + return ds + + +def write_null(parent: Any, name: str, replace: bool = False) -> Any: + """Write an explicit null, matching how anndata >= 0.12 encodes `None`. + + HDF5 uses a null dataspace (`h5py.Empty`); Zarr uses a 0-d boolean array. + Both carry `encoding-type: null`, so `None` round-trips instead of needing + an invented marker attribute. + """ + if replace: + _replace(parent, name) + + if is_zarr_group(parent): + ds = parent.create_array(name, shape=(), dtype=bool) + else: + import h5py + + ds = parent.create_dataset(name, data=h5py.Empty("f4")) + spec.set_encoding(ds, spec.NULL) + return ds + + +def write_categorical( + parent: Any, + name: str, + codes: Any, + categories: Sequence[Any], + ordered: bool = False, + replace: bool = False, +) -> Any: + """Write a `categorical` group of `categories` + `codes`. + + A code of -1 denotes a missing value, as the spec requires. + """ + if replace: + _replace(parent, name) + group = parent.create_group(name) + spec.set_encoding(group, spec.CATEGORICAL) + group.attrs["ordered"] = bool(ordered) + + write_string_array(group, "categories", categories) + write_dense(group, "codes", np.asarray(codes, dtype=_codes_dtype(len(categories)))) + return group + + +def _codes_dtype(n_categories: int) -> Any: + """Smallest signed integer dtype that can index `n_categories` plus -1.""" + if n_categories < 128: + return np.int8 + if n_categories < 32_768: + return np.int16 + return np.int32 + + +def write_masked( + parent: Any, + name: str, + values: Any, + mask: Any, + enc_type: str, + na_value: Optional[str] = None, + replace: bool = False, +) -> Any: + """Write a nullable element as a `values` + `mask` group. + + A True entry in `mask` marks a missing value. + """ + if enc_type not in spec.MASKED_TYPES: + raise ValueError(f"{enc_type!r} is not a masked encoding.") + if replace: + _replace(parent, name) + + group = parent.create_group(name) + spec.set_encoding(group, enc_type) + if na_value is not None: + group.attrs["na-value"] = na_value + + if enc_type == spec.NULLABLE_STRING_ARRAY: + write_string_array(group, "values", values) + else: + write_dense(group, "values", values) + write_dense(group, "mask", np.asarray(mask, dtype=bool)) + return group + + +def write_sparse( + parent: Any, + name: str, + data: Any, + indices: Any, + indptr: Any, + shape: Sequence[int], + enc_type: str = spec.CSR_MATRIX, + replace: bool = False, +) -> Any: + """Write a CSR/CSC sparse matrix group.""" + if enc_type not in spec.SPARSE_TYPES: + raise ValueError(f"{enc_type!r} is not a sparse encoding.") + if replace: + _replace(parent, name) + + group = parent.create_group(name) + spec.set_encoding(group, enc_type) + set_shape_attr(group, shape) + + create_dataset(group, "data", data=np.asarray(data)) + create_dataset(group, "indices", data=np.asarray(indices)) + create_dataset(group, "indptr", data=np.asarray(indptr)) + return group + + +def set_shape_attr(group: Any, shape: Sequence[int]) -> None: + """Set a sparse group's `shape`, in the form the backend's attrs accept. + + Zarr attributes must be JSON-serialisable, so a numpy array cannot be + stored there; HDF5 conventionally holds an integer array. + """ + dims = [int(d) for d in shape] + if is_zarr_group(group): + group.attrs["shape"] = dims + else: + group.attrs["shape"] = np.array(dims, dtype=np.int64) + + +def write_dataframe_header( + parent: Any, + name: str, + index_values: Iterable[Any], + column_order: Sequence[str], + index_name: str = "_index", + replace: bool = True, +) -> Any: + """Create a `dataframe` group with its index and declared column order. + + Columns themselves are written afterwards by the caller; `column-order` + records the authored order so readers do not fall back to the backend's + own (alphabetical, on HDF5) enumeration. + """ + if replace: + _replace(parent, name) + + group = parent.create_group(name) + spec.set_encoding(group, spec.DATAFRAME) + group.attrs["_index"] = index_name + set_column_order(group, column_order) + write_string_array(group, index_name, index_values) + return group + + +def set_column_order(group: Any, columns: Sequence[str]) -> None: + """Record a dataframe's column order in the form the backend accepts. + + Zarr attributes must be JSON, so a plain list is used. HDF5 needs an + explicit variable-length UTF-8 dtype -- an object array has no native + HDF5 equivalent, and an empty one cannot be inferred at all. + """ + names = [str(c) for c in columns] + if is_zarr_group(group): + group.attrs["column-order"] = names + else: + import h5py + + group.attrs["column-order"] = np.array( + names, dtype=h5py.string_dtype(encoding="utf-8") + ) + + +def ensure_anndata_skeleton(root: Any) -> None: + """Create the optional mapping groups an AnnData store is expected to have.""" + for key in ("layers", "obsm", "obsp", "varm", "varp", "uns"): + write_mapping(root, key) diff --git a/src/h5ad/formats/__init__.py b/src/adata/formats/__init__.py similarity index 100% rename from src/h5ad/formats/__init__.py rename to src/adata/formats/__init__.py diff --git a/src/h5ad/formats/array.py b/src/adata/formats/array.py similarity index 77% rename from src/h5ad/formats/array.py rename to src/adata/formats/array.py index 1dd21ac..07b6351 100644 --- a/src/h5ad/formats/array.py +++ b/src/adata/formats/array.py @@ -1,24 +1,32 @@ from __future__ import annotations from pathlib import Path -from typing import Any +import sys +from typing import Any, Optional import numpy as np from rich.console import Console -from h5ad.formats.common import _get_encoding_type, _resolve -from h5ad.formats.validate import validate_dimensions -from h5ad.storage import create_dataset, is_dataset, is_group -from h5ad.util.path import norm_path +from adata.formats.common import _get_encoding_type, _resolve +from adata.formats.validate import validate_dimensions +from adata.elements.write import write_dense +from adata.storage import create_dataset, is_dataset, is_group +from adata.util.path import norm_path def export_npy( root: Any, obj: str, - out: Path, + out: Optional[Path], chunk_elements: int, console: Console, ) -> None: + """Write a dense array to a .npy file, or to stdout when `out` is None. + + Writing to a file streams in chunks through a memory-mapped output. Stdout + is not seekable, so that path materialises the array first and is only + suitable for arrays that fit in memory. + """ h5obj = _resolve(root, obj) if is_group(h5obj): @@ -37,6 +45,11 @@ def export_npy( else: raise ValueError("Target is not an array-like object.") + if out is None or str(out) == "-": + np.save(sys.stdout.buffer, np.asarray(ds[...]), allow_pickle=False) + sys.stdout.buffer.flush() + return + out.parent.mkdir(parents=True, exist_ok=True) mm = np.lib.format.open_memmap(out, mode="w+", dtype=ds.dtype, shape=ds.shape) try: @@ -90,10 +103,7 @@ def import_npy( parent = parent[part] if part in parent else parent.create_group(part) name = parts[-1] - if name in parent: - del parent[name] - - create_dataset(parent, name, data=arr) + write_dense(parent, name, arr, replace=True) shape_str = "×".join(str(d) for d in arr.shape) console.print(f"[green]Imported[/] {shape_str} array into '{obj}'") diff --git a/src/h5ad/formats/common.py b/src/adata/formats/common.py similarity index 73% rename from src/h5ad/formats/common.py rename to src/adata/formats/common.py index 6282eb5..8333934 100644 --- a/src/h5ad/formats/common.py +++ b/src/adata/formats/common.py @@ -1,27 +1,11 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any import numpy as np -from h5ad.storage import is_dataset, is_group -from h5ad.util.path import norm_path - - -TYPE_EXTENSIONS = { - "dataframe": {".csv"}, - "sparse-matrix": {".mtx"}, - "dense-matrix": {".npy", ".png", ".jpg", ".jpeg", ".tif", ".tiff"}, - "array": {".npy", ".png", ".jpg", ".jpeg", ".tif", ".tiff"}, - "dict": {".json"}, - "scalar": {".json"}, - "categorical": {".csv"}, - "awkward-array": {".json"}, -} - -IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".tif", ".tiff"} - -EXPORTABLE_TYPES = set(TYPE_EXTENSIONS.keys()) +from adata.storage import is_dataset, is_group +from adata.util.path import norm_path def _get_encoding_type(group: Any) -> str: diff --git a/src/adata/formats/dataframe.py b/src/adata/formats/dataframe.py new file mode 100644 index 0000000..93110b7 --- /dev/null +++ b/src/adata/formats/dataframe.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import csv +import sys +from contextlib import nullcontext +from pathlib import Path +from typing import Any, List, Optional, Tuple + +import numpy as np +from rich.console import Console + +from adata.core.read import col_chunk_as_strings +from adata.formats.common import _resolve +from adata.elements.read import dataframe_columns, element_len, resolve_index +from adata.formats.validate import validate_dimensions +from adata.elements.write import ( + write_categorical, + write_dataframe_header, + write_dense, + write_string_array, +) +from adata.storage import create_dataset, is_group, is_zarr_group + + +def export_dataframe( + root: Any, + axis: str, + columns: Optional[List[str]], + out: Optional[Path], + chunk_rows: int, + head: Optional[int], + console: Console, +) -> None: + """Stream a dataframe group out as CSV. + + `axis` is any path to a dataframe-encoded group, not only "obs" or "var" -- + a store may hold dataframes in obsm, varm, uns or raw as well. + """ + group = _resolve(root, axis) + if not is_group(group): + raise ValueError(f"'{axis}' is not a group and cannot be exported as CSV.") + + try: + index, index_name = resolve_index(group) + except KeyError as exc: + raise ValueError( + f"'{axis}' is not a dataframe: it has no index. " + "CSV export needs a dataframe-encoded group such as 'obs', 'var' " + "or 'raw/var'." + ) from exc + n_rows = element_len(index) + + if isinstance(index_name, bytes): + index_name = index_name.decode("utf-8") + + if columns: + col_names = list(columns) + else: + col_names = dataframe_columns(group, index_name) + + if index_name not in col_names: + col_names.insert(0, index_name) + else: + col_names = [index_name] + [c for c in col_names if c != index_name] + + if head is not None and head > 0: + n_rows = min(n_rows, head) + + if out is None or str(out) == "-": + out_fh = sys.stdout + else: + out_fh = open(out, "w", newline="", encoding="utf-8") + writer = csv.writer(out_fh) + + try: + writer.writerow(col_names) + cat_cache = {} + + use_status = out_fh is not sys.stdout + status_ctx = ( + console.status(f"[magenta]Exporting {axis} table to {out}...[/]") + if use_status + else nullcontext() + ) + + with status_ctx as status: + for start in range(0, n_rows, chunk_rows): + end = min(start + chunk_rows, n_rows) + if use_status and status: + status.update( + f"[magenta]Exporting rows {start}-{end} of {n_rows}...[/]" + ) + cols_data: List[List[str]] = [] + for col in col_names: + cols_data.append( + col_chunk_as_strings(group, col, start, end, cat_cache) + ) + for row_idx in range(end - start): + row = [ + cols_data[col_idx][row_idx] + for col_idx in range(len(col_names)) + ] + writer.writerow(row) + finally: + if out_fh is not sys.stdout: + out_fh.close() + + +def _read_csv( + input_file: Path, + index_column: Optional[str], +) -> Tuple[List[dict], List[str], List[str], str]: + with open(input_file, "r", encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + if reader.fieldnames is None: + raise ValueError("CSV file has no header.") + fieldnames = list(reader.fieldnames) + + if index_column: + if index_column not in fieldnames: + raise ValueError( + f"Index column '{index_column}' not found in CSV. " + f"Available columns: {', '.join(fieldnames)}" + ) + idx_col = index_column + else: + idx_col = fieldnames[0] + + rows = list(reader) + + index_values = [row[idx_col] for row in rows] + data_columns = [c for c in fieldnames if c != idx_col] + + return rows, data_columns, index_values, idx_col + + +def _looks_categorical(values: List[str], n_rows: int) -> bool: + """Heuristic: few enough repeated labels that a category is the better fit. + + Mirrors the usual pandas rule of thumb -- a column whose distinct values + number well under half its rows is almost always a label, not free text. + Callers can always override explicitly. + """ + n_distinct = len(set(values)) + return 0 < n_distinct <= min(1000, max(1, n_rows // 2)) + + +def _write_column( + group: Any, + name: str, + values: List[str], + n_rows: int, + categorical: bool, +) -> None: + """Write one CSV column, choosing the narrowest faithful encoding.""" + try: + write_dense(group, name, np.array(values, dtype=np.int64)) + return + except (ValueError, TypeError, OverflowError): + pass + + try: + write_dense(group, name, np.array(values, dtype=np.float64)) + return + except (ValueError, TypeError): + pass + + if categorical: + categories = sorted(set(values)) + lookup = {c: i for i, c in enumerate(categories)} + write_categorical( + group, name, [lookup[v] for v in values], categories, ordered=False + ) + return + + write_string_array(group, name, values) + + +def import_dataframe( + root: Any, + obj: str, + input_file: Path, + index_column: Optional[str], + console: Console, + categorical: Optional[List[str]] = None, + auto_categorical: bool = True, +) -> None: + """Replace `obs` or `var` with the contents of a CSV file. + + Columns that parse cleanly as integers or floats become numeric arrays; + the rest become either a `categorical` or a `string-array`. `categorical` + names columns to force, and `auto_categorical` applies the heuristic to + the remainder. + """ + if obj not in ("obs", "var"): + raise ValueError( + f"CSV import is only supported for 'obs' or 'var', not '{obj}'." + ) + + rows, data_columns, index_values, _ = _read_csv(input_file, index_column) + n_rows = len(rows) + + validate_dimensions(root, obj, (n_rows,), console) + + index_name = "_index" + group = write_dataframe_header( + root, obj, index_values, data_columns, index_name=index_name + ) + + forced = set(categorical or ()) + n_categorical = 0 + for col in data_columns: + values = [row[col] for row in rows] + as_categorical = col in forced or ( + auto_categorical and _looks_categorical(values, n_rows) + ) + _write_column(group, col, values, n_rows, as_categorical) + if as_categorical and not _is_numeric(group, col): + n_categorical += 1 + + detail = f" ({n_categorical} categorical)" if n_categorical else "" + console.print( + f"[green]Imported[/] {n_rows} rows x {len(data_columns)} columns " + f"into '{obj}'{detail}" + ) + + +def _is_numeric(group: Any, col: str) -> bool: + """True when the written column ended up as a numeric array.""" + from adata.elements import spec + + return spec.encoding_type(group[col]) == spec.ARRAY diff --git a/src/h5ad/formats/image.py b/src/adata/formats/image.py similarity index 94% rename from src/h5ad/formats/image.py rename to src/adata/formats/image.py index fe5d2ce..e173325 100644 --- a/src/h5ad/formats/image.py +++ b/src/adata/formats/image.py @@ -7,8 +7,8 @@ from PIL import Image from rich.console import Console -from h5ad.formats.common import _resolve -from h5ad.storage import is_dataset +from adata.formats.common import _resolve +from adata.storage import is_dataset def export_image(root: Any, obj: str, out: Path, console: Console) -> None: diff --git a/src/h5ad/formats/json_data.py b/src/adata/formats/json_data.py similarity index 66% rename from src/h5ad/formats/json_data.py rename to src/adata/formats/json_data.py index c983677..8fb8c62 100644 --- a/src/h5ad/formats/json_data.py +++ b/src/adata/formats/json_data.py @@ -8,10 +8,17 @@ import numpy as np from rich.console import Console -from h5ad.core.read import decode_str_array -from h5ad.formats.common import _check_json_exportable, _resolve -from h5ad.storage import create_dataset, is_dataset, is_group -from h5ad.util.path import norm_path +from adata.core.read import decode_str_array +from adata.formats.common import _check_json_exportable, _resolve +from adata.elements.write import ( + write_dense, + write_mapping, + write_null, + write_scalar, + write_string_array, +) +from adata.storage import create_dataset, is_dataset, is_group +from adata.util.path import norm_path def export_json( @@ -128,28 +135,53 @@ def import_json( def _write_json_to_group(parent: Any, name: str, value: Any) -> None: + """Write one JSON value as the AnnData element that best represents it.""" if isinstance(value, dict): - group = parent.create_group(name) + group = write_mapping(parent, name, replace=True) for k, v in value.items(): _write_json_to_group(group, k, v) - elif isinstance(value, list): - try: - arr = np.array(value) - if arr.dtype.kind in ("U", "O"): - arr = np.array(value, dtype="S") - create_dataset(parent, name, data=arr) - except (ValueError, TypeError): - create_dataset(parent, name, data=json.dumps(value).encode("utf-8")) - elif isinstance(value, str): - create_dataset(parent, name, data=np.array([value], dtype="S")) - elif isinstance(value, bool): - create_dataset(parent, name, data=np.array(value, dtype=bool)) - elif isinstance(value, int): - create_dataset(parent, name, data=np.array(value, dtype=np.int64)) - elif isinstance(value, float): - create_dataset(parent, name, data=np.array(value, dtype=np.float64)) - elif value is None: - ds = create_dataset(parent, name, data=np.array([], dtype="S")) - ds.attrs["_is_none"] = True - else: - raise ValueError(f"Cannot convert JSON value of type {type(value).__name__}") + return + + if value is None: + write_null(parent, name, replace=True) + return + + if isinstance(value, str): + write_scalar(parent, name, value, replace=True) + return + + if isinstance(value, (bool, int, float)): + write_scalar(parent, name, value, replace=True) + return + + if isinstance(value, list): + _write_json_list(parent, name, value) + return + + raise ValueError(f"Cannot convert JSON value of type {type(value).__name__}") + + +def _write_json_list(parent: Any, name: str, value: list) -> None: + """Write a JSON array as a string or numeric array where it is uniform. + + Ragged or mixed lists have no array representation in the spec, so they + are stored as their JSON text rather than silently reshaped. + """ + if all(isinstance(v, str) for v in value): + write_string_array(parent, name, value, replace=True) + return + + try: + arr = np.array(value) + except (ValueError, TypeError): + arr = None + + if arr is not None and arr.dtype.kind in ("b", "i", "u", "f"): + write_dense(parent, name, arr, replace=True) + return + + if arr is not None and arr.dtype.kind in ("U", "S", "O", "T"): + write_string_array(parent, name, arr.reshape(-1).tolist(), replace=True) + return + + write_scalar(parent, name, json.dumps(value), replace=True) diff --git a/src/h5ad/formats/sparse.py b/src/adata/formats/sparse.py similarity index 92% rename from src/h5ad/formats/sparse.py rename to src/adata/formats/sparse.py index 4045ce5..37f0f42 100644 --- a/src/h5ad/formats/sparse.py +++ b/src/adata/formats/sparse.py @@ -8,10 +8,12 @@ import numpy as np from rich.console import Console -from h5ad.formats.common import _get_encoding_type, _resolve -from h5ad.formats.validate import validate_dimensions -from h5ad.storage import create_dataset, is_dataset, is_group, is_zarr_group -from h5ad.util.path import norm_path +from adata.elements import spec +from adata.elements.write import write_sparse +from adata.formats.common import _get_encoding_type, _resolve +from adata.formats.validate import validate_dimensions +from adata.storage import create_dataset, is_dataset, is_group, is_zarr_group +from adata.util.path import norm_path def _read_mtx( @@ -242,20 +244,9 @@ def import_mtx( parent = parent[part] if part in parent else parent.create_group(part) name = parts[-1] - if name in parent: - del parent[name] - - group = parent.create_group(name) - group.attrs["encoding-type"] = "csr_matrix" - group.attrs["encoding-version"] = "0.1.0" - if is_zarr_group(group): - group.attrs["shape"] = list(shape) - else: - group.attrs["shape"] = np.array(shape, dtype=np.int64) - - create_dataset(group, "data", data=data) - create_dataset(group, "indices", data=indices) - create_dataset(group, "indptr", data=indptr) + write_sparse( + parent, name, data, indices, indptr, shape, spec.CSR_MATRIX, replace=True + ) console.print( f"[green]Imported[/] {shape[0]}×{shape[1]} sparse matrix ({nnz} non-zero) into '{obj}'" diff --git a/src/h5ad/formats/validate.py b/src/adata/formats/validate.py similarity index 97% rename from src/h5ad/formats/validate.py rename to src/adata/formats/validate.py index 194192b..ce027d2 100644 --- a/src/h5ad/formats/validate.py +++ b/src/adata/formats/validate.py @@ -4,8 +4,8 @@ from rich.console import Console -from h5ad.core.info import axis_len -from h5ad.util.path import norm_path +from adata.core.info import axis_len +from adata.util.path import norm_path OBS_AXIS_PREFIXES = ("obs", "obsm/", "obsp/") diff --git a/src/adata/info.py b/src/adata/info.py new file mode 100644 index 0000000..cf39d1a --- /dev/null +++ b/src/adata/info.py @@ -0,0 +1,3 @@ +from adata.core.info import axis_len, format_type_info, get_axis_group, get_entry_type + +__all__ = ["axis_len", "format_type_info", "get_axis_group", "get_entry_type"] diff --git a/src/adata/read.py b/src/adata/read.py new file mode 100644 index 0000000..85d528b --- /dev/null +++ b/src/adata/read.py @@ -0,0 +1,19 @@ +from adata.core.read import ( + col_chunk_as_strings, + decode_str_array, + element_len, + read_categorical_column, + read_str_all, + read_str_chunk, + resolve_index, +) + +__all__ = [ + "col_chunk_as_strings", + "decode_str_array", + "element_len", + "read_categorical_column", + "read_str_all", + "read_str_chunk", + "resolve_index", +] diff --git a/src/h5ad/storage/__init__.py b/src/adata/storage/__init__.py similarity index 66% rename from src/h5ad/storage/__init__.py rename to src/adata/storage/__init__.py index 29c2227..929a95c 100644 --- a/src/h5ad/storage/__init__.py +++ b/src/adata/storage/__init__.py @@ -25,6 +25,7 @@ class Store: backend: str root: Any path: Path + zarr_format: Optional[int] = None def close(self) -> None: if self.backend == "hdf5": @@ -95,25 +96,54 @@ def detect_backend(path: Path) -> str: return "hdf5" -def open_store(path: Path, mode: str) -> Store: +def open_store( + path: Path, + mode: str, + zarr_format: Optional[int] = None, + require_anndata: bool = True, +) -> Store: + """Open a store, auto-detecting the backend from `path`. + + `zarr_format` selects the Zarr spec version for a store being created; + without it zarr-python uses its own default, which is v3. Callers writing a + derived store should pass the source store's format so a v2 input is not + silently upgraded. + + Set `require_anndata=False` for format-agnostic commands such as `ls`, + which are expected to open plain HDF5 and Zarr stores and should not warn + about a missing AnnData root. + """ path = Path(path) backend = detect_backend(path) if backend == "zarr": _require_zarr() - root = zarr.open_group(str(path), mode=mode) + kwargs = {} + if zarr_format is not None: + kwargs["zarr_format"] = zarr_format + root = zarr.open_group(str(path), mode=mode, **kwargs) if _is_writable_mode(mode): ensure_anndata_root_attrs(root) - else: + elif require_anndata: warn_if_missing_anndata_root_attrs(root, path=path) - return Store(backend="zarr", root=root, path=path) + return Store( + backend="zarr", + root=root, + path=path, + zarr_format=zarr_format_of(root), + ) root = h5py.File(path, mode) if _is_writable_mode(mode): ensure_anndata_root_attrs(root) - else: + elif require_anndata: warn_if_missing_anndata_root_attrs(root, path=path) return Store(backend="hdf5", root=root, path=path) +def zarr_format_of(obj: Any) -> Optional[int]: + """The Zarr spec version (2 or 3) backing `obj`, or None for HDF5.""" + return getattr(getattr(obj, "metadata", None), "zarr_format", None) + + def _decode_attr(value: Any) -> Any: if isinstance(value, bytes): return value.decode("utf-8") @@ -175,7 +205,16 @@ def copy_attrs(src_attrs: Any, dst_attrs: Any, *, target_backend: str) -> None: dst_attrs[k] = _normalize_attr_value(v, target_backend) -def dataset_create_kwargs(src: Any, *, target_backend: str) -> dict: +def dataset_create_kwargs( + src: Any, *, target_backend: str, zarr_format: Optional[int] = None +) -> dict: + """Derive creation kwargs that carry a source's layout onto a new dataset. + + Chunking, compression and sharding are preserved where the target backend + can express them; codecs that do not survive the crossing are dropped + rather than forwarded into an error. + """ + kw_target = zarr_format kw: dict = {} chunks = getattr(src, "chunks", None) if chunks is not None: @@ -211,8 +250,18 @@ def dataset_create_kwargs(src: Any, *, target_backend: str) -> dict: filters = getattr(src, "filters", None) except Exception: filters = None - if filters is not None: - kw["filters"] = filters + if filters: + # A v2 string array carries VLenUTF8 in `filters`; a v3 array + # rejects it (`Expected an ArrayArrayCodec`) because its string + # dtype encodes variable length itself. + if not (_target_zarr_format(kw_target) == 3 and _is_string_src(src)): + kw["filters"] = filters + try: + shards = getattr(src, "shards", None) + except Exception: + shards = None + if shards is not None: + kw["shards"] = shards try: fill_value = getattr(src, "fill_value", None) except Exception: @@ -222,6 +271,43 @@ def dataset_create_kwargs(src: Any, *, target_backend: str) -> dict: return kw +def _create_string_dataset( + parent: Any, + name: str, + data: Any, + **kwargs: Any, +) -> Any: + """Create a spec-compliant variable-length UTF-8 array from `data`. + + Text needs a different spelling on each backend and neither accepts the + other's: Zarr rejects the `object` dtype an h5py vlen dataset reports, and + h5py rejects Zarr's ` Any: + """Create an array under `parent`, backend-agnostically. + + String data is always written as variable-length UTF-8 regardless of how + it arrived, so callers can hand over bytes, ` Optional[int]: + return zarr_format if zarr_format is not None else 3 + + +def _is_string_src(src: Any) -> bool: + from adata.elements.strings import is_string_dtype + + return is_string_dtype(getattr(src, "dtype", None)) + + def _chunk_step(shape: Sequence[int], chunks: Optional[Sequence[int]]) -> int: if chunks is not None and len(chunks) > 0 and chunks[0]: return int(chunks[0]) @@ -260,14 +377,25 @@ def _chunk_step(shape: Sequence[int], chunks: Optional[Sequence[int]]) -> int: def copy_dataset(src: Any, dst_group: Any, name: str) -> Any: + """Copy a dataset into `dst_group`, streaming it in chunks. + + The destination dtype is resolved for the target backend rather than + reused: an h5py variable-length string dataset reports `object`, which Zarr + refuses to create, so copying one verbatim fails on every real store. + """ + from adata.elements.strings import target_dtype + shape = tuple(src.shape) if getattr(src, "shape", None) is not None else () target_backend = "zarr" if is_zarr_group(dst_group) else "hdf5" + zformat = zarr_format_of(dst_group) ds = create_dataset( dst_group, name, shape=shape, - dtype=src.dtype, - **dataset_create_kwargs(src, target_backend=target_backend), + dtype=target_dtype(src.dtype, target_backend, zformat), + **dataset_create_kwargs( + src, target_backend=target_backend, zarr_format=zformat + ), ) copy_attrs(src.attrs, ds.attrs, target_backend=target_backend) diff --git a/src/h5ad/util/__init__.py b/src/adata/util/__init__.py similarity index 100% rename from src/h5ad/util/__init__.py rename to src/adata/util/__init__.py diff --git a/src/h5ad/util/path.py b/src/adata/util/path.py similarity index 100% rename from src/h5ad/util/path.py rename to src/adata/util/path.py diff --git a/src/h5ad/commands/__init__.py b/src/h5ad/commands/__init__.py deleted file mode 100644 index 70d960f..0000000 --- a/src/h5ad/commands/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from h5ad.commands.info import show_info -from h5ad.commands.subset import subset_h5ad -from h5ad.commands.export import export_table, export_image, export_json, export_mtx, export_npy -from h5ad.commands.import_data import import_object diff --git a/src/h5ad/core/info.py b/src/h5ad/core/info.py deleted file mode 100644 index 8db8a14..0000000 --- a/src/h5ad/core/info.py +++ /dev/null @@ -1,221 +0,0 @@ -from __future__ import annotations - -from typing import Optional, Tuple, Dict, Any, Union - -import numpy as np - -from h5ad.storage import is_dataset, is_group, is_hdf5_dataset - - -def _decode_attr(value: Any) -> Any: - if isinstance(value, bytes): - return value.decode("utf-8") - return value - - -def get_entry_type(entry: Any) -> Dict[str, Any]: - """ - Determine the type/format of an object for export guidance. - - Supports both: - - v0.2.0 (modern): Objects with encoding-type/encoding-version attributes - - v0.1.0 (legacy): Objects without encoding attributes, inferred from structure - """ - result: Dict[str, Any] = { - "type": "unknown", - "export_as": None, - "encoding": None, - "shape": None, - "dtype": None, - "details": "", - "version": None, - } - - enc = _decode_attr(entry.attrs.get("encoding-type", b"")) - result["encoding"] = enc if enc else None - - enc_ver = _decode_attr(entry.attrs.get("encoding-version", b"")) - result["version"] = enc_ver if enc_ver else None - - if is_dataset(entry): - result["shape"] = entry.shape - result["dtype"] = str(entry.dtype) - - if "categories" in entry.attrs: - result["type"] = "categorical" - result["export_as"] = "csv" - result["version"] = result["version"] or "0.1.0" - n_cats = "?" - if is_hdf5_dataset(entry): - try: - cats_ref = entry.attrs["categories"] - cats_ds = entry.file[cats_ref] - n_cats = cats_ds.shape[0] - except Exception: - n_cats = "?" - result["details"] = ( - f"Legacy categorical [{entry.shape[0]} values, {n_cats} categories]" - ) - return result - - if entry.shape == (): - result["type"] = "scalar" - result["export_as"] = "json" - result["details"] = f"Scalar value ({entry.dtype})" - return result - - if entry.ndim == 1: - result["type"] = "array" - result["export_as"] = "npy" - result["details"] = f"1D array [{entry.shape[0]}] ({entry.dtype})" - elif entry.ndim == 2: - result["type"] = "dense-matrix" - result["export_as"] = "npy" - result["details"] = ( - f"Dense matrix {entry.shape[0]}×{entry.shape[1]} ({entry.dtype})" - ) - elif entry.ndim == 3: - result["type"] = "array" - result["export_as"] = "npy" - result["details"] = f"3D array {entry.shape} ({entry.dtype})" - else: - result["type"] = "array" - result["export_as"] = "npy" - result["details"] = f"ND array {entry.shape} ({entry.dtype})" - return result - - if is_group(entry): - if enc in ("csr_matrix", "csc_matrix"): - shape = entry.attrs.get("shape", None) - shape_str = f"{shape[0]}×{shape[1]}" if shape is not None else "?" - result["type"] = "sparse-matrix" - result["export_as"] = "mtx" - result["details"] = ( - f"Sparse {enc.replace('_matrix', '').upper()} matrix {shape_str}" - ) - return result - - if enc == "categorical": - codes = entry.get("codes") - cats = entry.get("categories") - n_codes = codes.shape[0] if codes is not None else "?" - n_cats = cats.shape[0] if cats is not None else "?" - result["type"] = "categorical" - result["export_as"] = "csv" - result["details"] = f"Categorical [{n_codes} values, {n_cats} categories]" - return result - - if ( - enc == "dataframe" - or "_index" in entry.attrs - or "obs_names" in entry - or "var_names" in entry - ): - if enc == "dataframe": - df_version = result["version"] or "0.2.0" - else: - df_version = "0.1.0" - result["version"] = df_version - - has_legacy_cats = "__categories" in entry - n_cols = len( - [k for k in entry.keys() if k not in ("_index", "__categories")] - ) - - result["type"] = "dataframe" - result["export_as"] = "csv" - if has_legacy_cats: - result["details"] = f"DataFrame with {n_cols} columns (legacy v0.1.0)" - else: - result["details"] = f"DataFrame with {n_cols} columns" - return result - - if enc in ("nullable-integer", "nullable-boolean", "nullable-string-array"): - result["type"] = "array" - result["export_as"] = "npy" - result["details"] = f"Encoded array ({enc})" - return result - - if enc == "string-array": - result["type"] = "array" - result["export_as"] = "npy" - result["details"] = "Encoded string array" - return result - - if enc == "awkward-array": - length = entry.attrs.get("length", "?") - result["type"] = "awkward-array" - result["export_as"] = "json" - result["details"] = f"Awkward array (length={length})" - return result - - n_keys = len(list(entry.keys())) - result["type"] = "dict" - result["export_as"] = "json" - result["details"] = f"Group with {n_keys} keys" - return result - - return result - - -def format_type_info(info: Dict[str, Any]) -> str: - type_colors = { - "dataframe": "green", - "sparse-matrix": "magenta", - "dense-matrix": "blue", - "array": "blue", - "dict": "yellow", - "categorical": "green", - "scalar": "white", - "unknown": "red", - } - - color = type_colors.get(info["type"], "white") - return f"[{color}]<{info['type']}>[/]" - - -def axis_len(file: Any, axis: str) -> int: - if axis not in file: - raise KeyError(f"'{axis}' not found in the file.") - - group = file[axis] - if not is_group(group): - raise TypeError(f"'{axis}' is not a group.") - - index_name = group.attrs.get("_index", None) - if index_name is None: - if axis == "obs": - index_name = "obs_names" - elif axis == "var": - index_name = "var_names" - else: - raise ValueError(f"Invalid axis '{axis}'. Must be 'obs' or 'var'.") - - index_name = _decode_attr(index_name) - - if index_name not in group: - raise KeyError(f"Index dataset '{index_name}' not found in '{axis}' group.") - - dataset = group[index_name] - if not is_dataset(dataset): - raise TypeError(f"Index '{index_name}' in '{axis}' is not a dataset.") - if dataset.shape: - return int(dataset.shape[0]) - raise ValueError( - f"Cannot determine length of '{axis}': index dataset has no shape." - ) - - -def get_axis_group(file: Any, axis: str) -> Tuple[Any, int, str]: - if axis not in ("obs", "var"): - raise ValueError("axis must be 'obs' or 'var'.") - - n = axis_len(file, axis) - group = file[axis] - - index_name = group.attrs.get("_index", None) - if index_name is None: - index_name = "obs_names" if axis == "obs" else "var_names" - index_name = _decode_attr(index_name) - - return group, n, index_name diff --git a/src/h5ad/core/read.py b/src/h5ad/core/read.py deleted file mode 100644 index 5c13e5e..0000000 --- a/src/h5ad/core/read.py +++ /dev/null @@ -1,142 +0,0 @@ -from __future__ import annotations - -from typing import List, Dict, Any - -import h5py -import numpy as np - -from h5ad.storage import is_group, is_dataset, is_hdf5_dataset - - -def decode_str_array(array: np.ndarray) -> np.ndarray: - arr = np.asarray(array) - - if np.issubdtype(arr.dtype, np.bytes_): - flat = arr.reshape(-1) - decoded = [ - ( - b.decode("utf-8", errors="replace") - if isinstance(b, (bytes, np.bytes_)) - else str(b) - ) - for b in flat - ] - return np.asarray(decoded, dtype=str).reshape(arr.shape) - - if arr.dtype.kind == "O": - flat = arr.reshape(-1) - decoded = [ - ( - v.decode("utf-8", errors="replace") - if isinstance(v, (bytes, np.bytes_)) - else str(v) - ) - for v in flat - ] - return np.asarray(decoded, dtype=str).reshape(arr.shape) - - return arr.astype(str) - - -def _categorical_cache_key(col: Any, parent_group: Any | None = None) -> str: - col_name = getattr(col, "name", None) - if isinstance(col_name, str) and col_name: - return col_name - - if parent_group is not None: - parent_name = getattr(parent_group, "name", "") - rel_name = getattr(col, "path", "") - if parent_name or rel_name: - return f"{parent_name}/{rel_name}" - - return repr(col) - - -def read_categorical_column( - col: Any, - start: int, - end: int, - cache: Dict[str, np.ndarray], - parent_group: Any | None = None, -) -> List[str]: - key = _categorical_cache_key(col, parent_group) - - if is_group(col): - if key not in cache: - cats = col["categories"][...] - cats = decode_str_array(cats) - cache[key] = np.asarray(cats, dtype=str) - cats = cache[key] - - codes_ds = col["codes"] - codes = codes_ds[start:end] - codes = np.asarray(codes, dtype=np.int64) - return [cats[c] if 0 <= c < len(cats) else "" for c in codes] - - if is_dataset(col): - if key not in cache: - cats_ref = col.attrs.get("categories", None) - if cats_ref is not None and is_hdf5_dataset(col): - cats_ds = col.file[cats_ref] - cats = cats_ds[...] - elif parent_group is not None and "__categories" in parent_group: - col_name = col.name.split("/")[-1] - cats_grp = parent_group["__categories"] - if col_name in cats_grp: - cats = cats_grp[col_name][...] - else: - raise KeyError( - f"Cannot find categories for legacy column {col.name}" - ) - else: - raise KeyError(f"Cannot find categories for legacy column {col.name}") - cats = decode_str_array(cats) - cache[key] = np.asarray(cats, dtype=str) - cats = cache[key] - - codes = col[start:end] - codes = np.asarray(codes, dtype=np.int64) - return [cats[c] if 0 <= c < len(cats) else "" for c in codes] - - raise TypeError(f"Unsupported categorical column type: {type(col)}") - - -def col_chunk_as_strings( - group: Any, - col_name: str, - start: int, - end: int, - cat_cache: Dict[str, np.ndarray], -) -> List[str]: - if col_name not in group: - raise RuntimeError(f"Column {col_name!r} not found in group {group.name}") - - col = group[col_name] - - if is_dataset(col): - if "categories" in col.attrs: - return read_categorical_column(col, start, end, cat_cache, group) - - chunk = col[start:end] - if chunk.ndim != 1: - chunk = chunk.reshape(-1) - chunk = decode_str_array(np.asarray(chunk)) - return chunk.tolist() - - if is_group(col): - enc = col.attrs.get("encoding-type", b"") - if isinstance(enc, bytes): - enc = enc.decode("utf-8") - - if enc == "categorical": - return read_categorical_column(col, start, end, cat_cache) - - if enc in ("nullable-integer", "nullable-boolean", "nullable-string-array"): - values = col["values"][start:end] - mask = col["mask"][start:end] - values = decode_str_array(np.asarray(values)) - return ["" if m else str(v) for v, m in zip(values, mask)] - - raise ValueError(f"Unsupported group encoding {enc!r} for column {col_name!r}") - - raise TypeError(f"Unsupported column type for {col_name!r} in group {group.name}") diff --git a/src/h5ad/formats/dataframe.py b/src/h5ad/formats/dataframe.py deleted file mode 100644 index f767c4c..0000000 --- a/src/h5ad/formats/dataframe.py +++ /dev/null @@ -1,169 +0,0 @@ -from __future__ import annotations - -import csv -import sys -from contextlib import nullcontext -from pathlib import Path -from typing import Any, List, Optional, Tuple - -import numpy as np -from rich.console import Console - -from h5ad.core.info import get_axis_group -from h5ad.core.read import col_chunk_as_strings -from h5ad.formats.validate import validate_dimensions -from h5ad.storage import create_dataset, is_zarr_group - - -def export_dataframe( - root: Any, - axis: str, - columns: Optional[List[str]], - out: Optional[Path], - chunk_rows: int, - head: Optional[int], - console: Console, -) -> None: - group, n_rows, index_name = get_axis_group(root, axis) - - reserved_keys = {"_index", "__categories"} - - if columns: - col_names = list(columns) - else: - col_names = [ - k for k in group.keys() if k not in reserved_keys and k != index_name - ] - if index_name and index_name not in col_names: - col_names.insert(0, index_name) - - if isinstance(index_name, bytes): - index_name = index_name.decode("utf-8") - - if index_name not in col_names: - col_names.insert(0, index_name) - else: - col_names = [index_name] + [c for c in col_names if c != index_name] - - if head is not None and head > 0: - n_rows = min(n_rows, head) - - if out is None or str(out) == "-": - out_fh = sys.stdout - else: - out_fh = open(out, "w", newline="", encoding="utf-8") - writer = csv.writer(out_fh) - - try: - writer.writerow(col_names) - cat_cache = {} - - use_status = out_fh is not sys.stdout - status_ctx = ( - console.status(f"[magenta]Exporting {axis} table to {out}...[/]") - if use_status - else nullcontext() - ) - - with status_ctx as status: - for start in range(0, n_rows, chunk_rows): - end = min(start + chunk_rows, n_rows) - if use_status and status: - status.update( - f"[magenta]Exporting rows {start}-{end} of {n_rows}...[/]" - ) - cols_data: List[List[str]] = [] - for col in col_names: - cols_data.append( - col_chunk_as_strings(group, col, start, end, cat_cache) - ) - for row_idx in range(end - start): - row = [ - cols_data[col_idx][row_idx] - for col_idx in range(len(col_names)) - ] - writer.writerow(row) - finally: - if out_fh is not sys.stdout: - out_fh.close() - - -def _read_csv( - input_file: Path, - index_column: Optional[str], -) -> Tuple[List[dict], List[str], List[str], str]: - with open(input_file, "r", encoding="utf-8", newline="") as f: - reader = csv.DictReader(f) - if reader.fieldnames is None: - raise ValueError("CSV file has no header.") - fieldnames = list(reader.fieldnames) - - if index_column: - if index_column not in fieldnames: - raise ValueError( - f"Index column '{index_column}' not found in CSV. " - f"Available columns: {', '.join(fieldnames)}" - ) - idx_col = index_column - else: - idx_col = fieldnames[0] - - rows = list(reader) - - index_values = [row[idx_col] for row in rows] - data_columns = [c for c in fieldnames if c != idx_col] - - return rows, data_columns, index_values, idx_col - - -def import_dataframe( - root: Any, - obj: str, - input_file: Path, - index_column: Optional[str], - console: Console, -) -> None: - if obj not in ("obs", "var"): - raise ValueError( - f"CSV import is only supported for 'obs' or 'var', not '{obj}'." - ) - - rows, data_columns, index_values, _ = _read_csv(input_file, index_column) - n_rows = len(rows) - - validate_dimensions(root, obj, (n_rows,), console) - - if obj in root: - del root[obj] - - group = root.create_group(obj) - index_name = "obs_names" if obj == "obs" else "var_names" - group.attrs["_index"] = index_name - group.attrs["encoding-type"] = "dataframe" - group.attrs["encoding-version"] = "0.2.0" - - if is_zarr_group(group): - group.attrs["column-order"] = list(data_columns) - else: - group.attrs["column-order"] = np.array(data_columns, dtype="S") - - create_dataset(group, index_name, data=np.array(index_values, dtype="S")) - - for col in data_columns: - values = [row[col] for row in rows] - try: - arr = np.array(values, dtype=np.float64) - create_dataset(group, col, data=arr) - except (ValueError, TypeError): - try: - arr = np.array(values, dtype=np.int64) - create_dataset(group, col, data=arr) - except (ValueError, TypeError): - arr = np.array(values, dtype="S") - ds = create_dataset(group, col, data=arr) - ds.attrs["encoding-type"] = "string-array" - ds.attrs["encoding-version"] = "0.2.0" - - console.print( - f"[green]Imported[/] {n_rows} rows × {len(data_columns)} columns into '{obj}'" - ) diff --git a/src/h5ad/info.py b/src/h5ad/info.py deleted file mode 100644 index 635b03a..0000000 --- a/src/h5ad/info.py +++ /dev/null @@ -1,3 +0,0 @@ -from h5ad.core.info import axis_len, format_type_info, get_axis_group, get_entry_type - -__all__ = ["axis_len", "format_type_info", "get_axis_group", "get_entry_type"] diff --git a/src/h5ad/read.py b/src/h5ad/read.py deleted file mode 100644 index 63f8c4d..0000000 --- a/src/h5ad/read.py +++ /dev/null @@ -1,3 +0,0 @@ -from h5ad.core.read import col_chunk_as_strings, decode_str_array, read_categorical_column - -__all__ = ["col_chunk_as_strings", "decode_str_array", "read_categorical_column"] diff --git a/tests/test_anndata_roundtrip.py b/tests/test_anndata_roundtrip.py new file mode 100644 index 0000000..f6f7fab --- /dev/null +++ b/tests/test_anndata_roundtrip.py @@ -0,0 +1,283 @@ +"""Interoperability tests against the real anndata library. + +These are the tests that matter most. Everything else in the suite checks the +CLI against stores the suite itself built, which cannot catch the case that +actually broke: anndata changing how it writes a file. Here anndata writes the +fixtures and reads the results back, in both directions and on both backends. + +Skipped when anndata is not installed. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +from typer.testing import CliRunner + +from adata.cli import app + +ad = pytest.importorskip("anndata", reason="anndata is required for round-trip tests") +pd = pytest.importorskip("pandas") +sparse = pytest.importorskip("scipy.sparse") + +runner = CliRunner() + +N_OBS, N_VAR = 6, 4 + + +def _build(tmp_path: Path, fmt: str, *, nullable_strings: bool) -> Path: + """Write a reference store with anndata covering every common encoding. + + `nullable_strings` selects between the two on-disk shapes anndata can give + a string column -- a plain `string-array` dataset, or the + `nullable-string-array` group it writes by default since 0.11. The index + itself follows the same choice, which is what broke the reader. + """ + obs = pd.DataFrame( + { + "cell_type": pd.Categorical( + ["A", "B", "A", "C", "B", "A"], categories=["A", "B", "C"], ordered=True + ), + "n_counts": np.arange(N_OBS, dtype="int32"), + "nullable_int": pd.array([1, 2, None, 4, 5, None], dtype="Int32"), + "nullable_bool": pd.array( + [True, False, None, True, False, None], dtype="boolean" + ), + "free_text": ["a", "bb", "ccc", "d", "ee", "f"], + }, + index=[f"cell_{i}" for i in range(N_OBS)], + ) + var = pd.DataFrame( + { + "gene_ids": [f"ENSG{i}" for i in range(N_VAR)], + "highly_variable": [True, False, True, False], + }, + index=[f"gene_{i}" for i in range(N_VAR)], + ) + X = sparse.csr_matrix( + np.random.default_rng(0).poisson(1.0, (N_OBS, N_VAR)).astype("float32") + ) + + obj = ad.AnnData(X=X, obs=obs, var=var) + obj.layers["counts"] = X.copy() + obj.obsm["X_pca"] = np.zeros((N_OBS, 3), dtype="float32") + obj.varm["PCs"] = np.zeros((N_VAR, 3), dtype="float32") + obj.obsp["connectivities"] = sparse.csr_matrix(np.eye(N_OBS, dtype="float32")) + obj.uns["a_string"] = "hello" + obj.uns["an_int"] = 42 + obj.uns["nested"] = {"k": np.arange(5)} + obj.raw = obj + + previous = ad.settings.allow_write_nullable_strings + ad.settings.allow_write_nullable_strings = nullable_strings + try: + path = tmp_path / f"ref.{fmt}" + if fmt == "h5ad": + obj.write_h5ad(path) + else: + obj.write_zarr(path) + finally: + ad.settings.allow_write_nullable_strings = previous + return path + + +def _read(path: Path): + """Read a store back with anndata, treating its format complaints as errors. + + anndata raises OldFormatWarning for elements missing encoding metadata, so + promoting it here means a store we write that is not fully spec-compliant + fails the test rather than merely warning. + """ + import warnings + + from anndata._warnings import OldFormatWarning + + with warnings.catch_warnings(): + warnings.simplefilter("error", OldFormatWarning) + return ad.read_zarr(path) if path.suffix == ".zarr" else ad.read_h5ad(path) + + +@pytest.fixture(params=["h5ad", "zarr"]) +def fmt(request) -> str: + return request.param + + +@pytest.fixture(params=[True, False], ids=["nullable-strings", "string-arrays"]) +def reference(request, fmt, tmp_path) -> Path: + return _build(tmp_path, fmt, nullable_strings=request.param) + + +def test_view_reads_anndata_output(reference): + """`view` must not choke on a file anndata just wrote.""" + result = runner.invoke(app, ["view", str(reference)]) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + assert f"{N_OBS} × {N_VAR}" in result.stdout + + +def test_ls_reads_anndata_output(reference): + result = runner.invoke(app, ["ls", str(reference), "--long"]) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + +def test_export_dataframe_reads_every_column(reference): + """Every obs column must render, whatever encoding anndata chose for it.""" + result = runner.invoke(app, ["export", "dataframe", str(reference), "obs"]) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + header, *rows = [ln for ln in result.stdout.splitlines() if ln] + assert header.split(",") == [ + "_index", + "cell_type", + "n_counts", + "nullable_int", + "nullable_bool", + "free_text", + ], "column-order should be honoured, not the backend's own ordering" + assert len(rows) == N_OBS + # Masked entries render empty rather than as a sentinel. + assert rows[2].split(",")[3] == "" + + +def test_export_dataframe_accepts_arbitrary_paths(reference): + """Dataframes outside obs/var are exportable too (issue #4).""" + result = runner.invoke(app, ["export", "dataframe", str(reference), "raw/var"]) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + assert "gene_ids" in result.stdout + + +@pytest.mark.parametrize("out_fmt", ["h5ad", "zarr"]) +def test_subset_roundtrips_through_anndata(reference, out_fmt, tmp_path): + """A subset must be readable by anndata with every dtype intact. + + Covers all four backend pairings, including the HDF5<->Zarr crossings + where a string dtype has to be translated rather than copied. + """ + names = tmp_path / "keep.txt" + names.write_text("cell_0\ncell_2\n") + out = tmp_path / f"subset.{out_fmt}" + + result = runner.invoke( + app, ["subset", str(reference), "-o", str(out), "--obs", str(names)] + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + got = _read(out) + assert got.shape == (2, N_VAR) + assert list(got.obs_names) == ["cell_0", "cell_2"] + + dtypes = got.obs.dtypes.astype(str).to_dict() + assert dtypes["cell_type"] == "category" + assert dtypes["nullable_int"] == "Int32" + assert dtypes["nullable_bool"] == "boolean" + assert got.obs["cell_type"].cat.ordered is True + assert list(got.obs["cell_type"].cat.categories) == ["A", "B", "C"] + + assert got.obs["nullable_int"].isna().tolist() == [False, True] + assert got.raw is not None, "raw/ must survive subsetting" + assert got.raw.shape == (2, N_VAR) + assert "counts" in got.layers + + +def test_subset_matches_var_on_raws_own_axis(reference, tmp_path): + """raw/ carries its own var axis and must be matched against it.""" + names = tmp_path / "vkeep.txt" + names.write_text("gene_0\ngene_2\n") + out = tmp_path / "subset_var.h5ad" + + result = runner.invoke( + app, ["subset", str(reference), "-o", str(out), "--var", str(names)] + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + got = _read(out) + assert list(got.var_names) == ["gene_0", "gene_2"] + assert list(got.raw.var_names) == ["gene_0", "gene_2"] + + +def test_csv_roundtrip_preserves_categoricals(reference, tmp_path): + """obs -> CSV -> obs must not degrade a categorical into free text.""" + csv = tmp_path / "obs.csv" + out = tmp_path / "reimported.h5ad" + + assert ( + runner.invoke( + app, ["export", "dataframe", str(reference), "obs", "-o", str(csv)] + ).exit_code + == 0 + ) + result = runner.invoke( + app, + ["import", "dataframe", str(reference), "obs", str(csv), + "-o", str(out), "-i", "_index"], + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + got = _read(out) + assert str(got.obs["cell_type"].dtype) == "category" + assert list(got.obs["cell_type"]) == ["A", "B", "A", "C", "B", "A"] + + +def test_json_roundtrip_covers_every_scalar_kind(reference, tmp_path): + """uns values must come back as the Python types they went in as.""" + payload = tmp_path / "payload.json" + payload.write_text( + '{"title":"run","n":100,"rate":0.5,"flag":true,"nothing":null,' + '"labels":["a","b"],"weights":[1.5,2.5]}' + ) + out = tmp_path / "with_uns.h5ad" + + result = runner.invoke( + app, + ["import", "dict", str(reference), "uns/run", str(payload), "-o", str(out)], + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + run = _read(out).uns["run"] + assert run["title"] == "run" + assert int(run["n"]) == 100 + assert float(run["rate"]) == pytest.approx(0.5) + assert bool(run["flag"]) is True + assert run["nothing"] is None + assert list(run["labels"]) == ["a", "b"] + assert list(run["weights"]) == pytest.approx([1.5, 2.5]) + + +@pytest.mark.parametrize("layout", ["csr", "csc"]) +def test_sparse_subset_matches_scipy(tmp_path, layout): + """Block-streamed sparse subsetting must equal scipy's own result exactly. + + Uses a matrix large enough to span several blocks, so the block boundary + handling and the minor-axis remap are actually exercised. + """ + rng = np.random.default_rng(7) + n, m = 2000, 800 + X = sparse.random(n, m, density=0.05, format="csr", dtype="float32", random_state=7) + + obj = ad.AnnData( + X=X if layout == "csr" else X.tocsc(), + obs=pd.DataFrame(index=[f"c{i}" for i in range(n)]), + var=pd.DataFrame(index=[f"g{i}" for i in range(m)]), + ) + src = tmp_path / "big.h5ad" + obj.write_h5ad(src) + + keep_obs = sorted(rng.choice(n, 500, replace=False)) + keep_var = sorted(rng.choice(m, 300, replace=False)) + (tmp_path / "obs.txt").write_text("\n".join(f"c{i}" for i in keep_obs)) + (tmp_path / "var.txt").write_text("\n".join(f"g{i}" for i in keep_var)) + + out = tmp_path / "big_subset.h5ad" + result = runner.invoke( + app, + ["subset", str(src), "-o", str(out), + "--obs", str(tmp_path / "obs.txt"), "--var", str(tmp_path / "var.txt")], + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + got = ad.read_h5ad(out) + expected = obj.X[keep_obs][:, keep_var] + assert got.shape == (len(keep_obs), len(keep_var)) + assert abs(got.X - expected).nnz == 0 + assert type(got.X).__name__ == f"{layout}_matrix" diff --git a/tests/test_cli.py b/tests/test_cli.py index 2fac5c1..9bbb49d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,38 +6,38 @@ import h5py import numpy as np from typer.testing import CliRunner -from h5ad.cli import app -from h5ad.commands.info import show_info -from h5ad.commands.export import export_table +from adata.cli import app +from adata.commands.info import show_info +from adata.commands.export import export_table from rich.console import Console runner = CliRunner() -class TestInfoCommand: +class TestViewCommand: """Tests for info command.""" - def test_info_command_success(self, sample_h5ad_file): + def test_view_command_success(self, sample_h5ad_file): """Test info command on valid file.""" - result = runner.invoke(app, ["info", str(sample_h5ad_file)]) + result = runner.invoke(app, ["view", str(sample_h5ad_file)]) assert result.exit_code == 0 assert "5 × 4" in result.stdout - def test_info_command_nonexistent_file(self): + def test_view_command_nonexistent_file(self): """Test info command on non-existent file.""" - result = runner.invoke(app, ["info", "nonexistent.h5ad"]) + result = runner.invoke(app, ["view", "nonexistent.h5ad"]) assert result.exit_code != 0 - def test_info_function_direct(self, sample_h5ad_file): + def test_view_function_direct(self, sample_h5ad_file): """Test show_info function directly.""" console = Console(stderr=True) # Should not raise exception show_info(sample_h5ad_file, console) - def test_info_tree_flag(self, sample_h5ad_file): + def test_view_tree_flag(self, sample_h5ad_file): """Test info command with --tree flag.""" - result = runner.invoke(app, ["info", "--tree", str(sample_h5ad_file)]) + result = runner.invoke(app, ["view", "--tree", str(sample_h5ad_file)]) assert result.exit_code == 0 # Should show type annotations in angle brackets # Output may go to stdout or stderr depending on console config @@ -45,55 +45,55 @@ def test_info_tree_flag(self, sample_h5ad_file): assert "<" in output assert ">" in output - def test_info_tree_short_flag(self, sample_h5ad_file): + def test_view_tree_short_flag(self, sample_h5ad_file): """Test info command with -t short flag.""" - result = runner.invoke(app, ["info", "-t", str(sample_h5ad_file)]) + result = runner.invoke(app, ["view", "-t", str(sample_h5ad_file)]) assert result.exit_code == 0 output = result.stdout + (result.stderr or "") assert "<" in output - def test_info_depth_flag(self, sample_h5ad_file): + def test_view_depth_flag(self, sample_h5ad_file): """Test info command with --depth flag.""" result = runner.invoke( - app, ["info", "--tree", "--depth", "1", str(sample_h5ad_file)] + app, ["view", "--tree", "--depth", "1", str(sample_h5ad_file)] ) assert result.exit_code == 0 output = result.stdout + (result.stderr or "") assert "<" in output - def test_info_depth_short_flag(self, sample_h5ad_file): + def test_view_depth_short_flag(self, sample_h5ad_file): """Test info command with -d short flag.""" - result = runner.invoke(app, ["info", "-t", "-d", "2", str(sample_h5ad_file)]) + result = runner.invoke(app, ["view", "-t", "-d", "2", str(sample_h5ad_file)]) assert result.exit_code == 0 output = result.stdout + (result.stderr or "") assert "<" in output - def test_info_entry_positional(self, sample_h5ad_file): + def test_view_entry_positional(self, sample_h5ad_file): """Test info command with entry as positional argument.""" - result = runner.invoke(app, ["info", str(sample_h5ad_file), "X"]) + result = runner.invoke(app, ["view", str(sample_h5ad_file), "X"]) assert result.exit_code == 0 output = result.stdout + (result.stderr or "") assert "Path:" in output assert "Type:" in output - def test_info_entry_obs(self, sample_h5ad_file): + def test_view_entry_obs(self, sample_h5ad_file): """Test info command with obs entry.""" - result = runner.invoke(app, ["info", str(sample_h5ad_file), "obs"]) + result = runner.invoke(app, ["view", str(sample_h5ad_file), "obs"]) assert result.exit_code == 0 output = result.stdout + (result.stderr or "") assert "Path:" in output assert "dataframe" in output - def test_info_entry_nested_path(self, sample_h5ad_file): + def test_view_entry_nested_path(self, sample_h5ad_file): """Test info command with nested object path.""" - result = runner.invoke(app, ["info", str(sample_h5ad_file), "uns/description"]) + result = runner.invoke(app, ["view", str(sample_h5ad_file), "uns/description"]) assert result.exit_code == 0 output = result.stdout + (result.stderr or "") assert "Path:" in output - def test_info_entry_not_found(self, sample_h5ad_file): + def test_view_entry_not_found(self, sample_h5ad_file): """Test info command with non-existent object path.""" - result = runner.invoke(app, ["info", str(sample_h5ad_file), "nonexistent"]) + result = runner.invoke(app, ["view", str(sample_h5ad_file), "nonexistent"]) assert result.exit_code == 0 # Doesn't exit with error, just shows message output = result.stdout + (result.stderr or "") assert "not found" in output @@ -393,7 +393,7 @@ def test_export_dataframe_multi_categorical_columns(self, temp_dir): assert rows[3] == ["cell3", "11.0", "BRC2243", "β-cell"] assert rows[4] == ["cell4", "nan", "nan", "nan"] - def test_export_dataframe_invalid_axis(self, sample_h5ad_file, temp_dir): + def test_export_dataframe_missing_entry(self, sample_h5ad_file, temp_dir): """Test export dataframe with invalid axis.""" output = temp_dir / "table.csv" result = runner.invoke( @@ -410,7 +410,7 @@ def test_export_dataframe_invalid_axis(self, sample_h5ad_file, temp_dir): assert result.exit_code == 1 # Check both stdout and stderr since Console uses stderr=True output_text = result.stdout + result.stderr - assert "obs" in output_text or "var" in output_text + assert "not found" in output_text def test_export_table_function(self, sample_h5ad_file, temp_dir): """Test export_table function directly.""" @@ -565,12 +565,18 @@ def test_cli_help(self): assert result.exit_code == 0 assert "Streaming CLI" in result.stdout - def test_info_help(self): - """Test info command help.""" - result = runner.invoke(app, ["info", "--help"]) + def test_view_help(self): + """Test view command help.""" + result = runner.invoke(app, ["view", "--help"]) assert result.exit_code == 0 assert "Show high-level information" in result.stdout + def test_info_alias_is_deprecated(self): + """The `info` alias still works but warns.""" + result = runner.invoke(app, ["info", "--help"]) + assert result.exit_code == 0 + assert "Deprecated alias" in result.stdout + def test_export_help(self): """Test export command help.""" result = runner.invoke(app, ["export", "--help"]) @@ -582,7 +588,7 @@ def test_export_dataframe_help(self): """Test export dataframe command help.""" result = runner.invoke(app, ["export", "dataframe", "--help"]) assert result.exit_code == 0 - assert "Export a dataframe" in result.stdout + assert "Export any dataframe" in result.stdout def test_import_help(self): """Test import command help.""" @@ -595,4 +601,4 @@ def test_subset_help(self): """Test subset command help.""" result = runner.invoke(app, ["subset", "--help"]) assert result.exit_code == 0 - assert "Subset an h5ad" in result.stdout + assert "Subset an AnnData store" in result.stdout diff --git a/tests/test_export.py b/tests/test_export.py index 6a3fad9..f796697 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -7,7 +7,7 @@ import numpy as np from typer.testing import CliRunner -from h5ad.cli import app +from adata.cli import app runner = CliRunner() @@ -226,7 +226,7 @@ def test_wrong_type_for_dataframe(self, sample_h5ad_file, temp_dir): ["export", "dataframe", str(sample_h5ad_file), "X", "--output", str(out)], ) assert result.exit_code == 1 - assert "obs" in result.output or "var" in result.output + assert "not a group" in result.output def test_sparse_matrix_array_export(self, sample_sparse_csr_h5ad, temp_dir): """Test that sparse matrix requires sparse export.""" diff --git a/tests/test_import.py b/tests/test_import.py index f49af84..b15fc9c 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -8,7 +8,7 @@ import numpy as np from typer.testing import CliRunner -from h5ad.cli import app +from adata.cli import app runner = CliRunner() diff --git a/tests/test_info_read.py b/tests/test_info_read.py index adc7378..8a28ba6 100644 --- a/tests/test_info_read.py +++ b/tests/test_info_read.py @@ -3,8 +3,8 @@ import pytest import h5py import numpy as np -from h5ad.info import axis_len, get_axis_group, get_entry_type, format_type_info -from h5ad.read import decode_str_array, read_categorical_column, col_chunk_as_strings +from adata.info import axis_len, get_axis_group, get_entry_type, format_type_info +from adata.read import decode_str_array, read_categorical_column, col_chunk_as_strings class TestGetEntryType: @@ -122,7 +122,7 @@ def test_axis_len_missing_index(self, temp_dir): with h5py.File(file_path, "w") as f: f.create_group("obs") with h5py.File(file_path, "r") as f: - with pytest.raises(KeyError, match="Index dataset 'obs_names' not found"): + with pytest.raises(KeyError, match="Could not find an index"): axis_len(f, "obs") diff --git a/tests/test_storage_root_attrs.py b/tests/test_storage_root_attrs.py index 13f4621..4d6e230 100644 --- a/tests/test_storage_root_attrs.py +++ b/tests/test_storage_root_attrs.py @@ -5,7 +5,7 @@ import h5py import pytest -from h5ad.storage import open_store +from adata.storage import open_store def _make_minimal_h5ad(path: Path) -> None: @@ -25,7 +25,7 @@ def test_open_store_read_warns_for_missing_root_attrs(temp_dir: Path) -> None: file_path = temp_dir / "missing_root_attrs.h5ad" _make_minimal_h5ad(file_path) - with pytest.warns(UserWarning, match="missing required AnnData attrs"): + with pytest.warns(UserWarning, match="missing or invalid AnnData attrs"): with open_store(file_path, "r"): pass diff --git a/tests/test_subset.py b/tests/test_subset.py index 42d5ec6..ef63609 100644 --- a/tests/test_subset.py +++ b/tests/test_subset.py @@ -4,7 +4,7 @@ import h5py import numpy as np from pathlib import Path -from h5ad.commands.subset import ( +from adata.commands.subset import ( _read_name_file, indices_from_name_set, subset_axis_group, diff --git a/tests/test_zarr.py b/tests/test_zarr.py index 1008d6a..527e5fa 100644 --- a/tests/test_zarr.py +++ b/tests/test_zarr.py @@ -10,8 +10,8 @@ from typer.testing import CliRunner from rich.console import Console -from h5ad.cli import app -from h5ad.core.subset import subset_h5ad +from adata.cli import app +from adata.core.subset import subset_h5ad zarr = pytest.importorskip("zarr") @@ -105,7 +105,7 @@ def test_info_zarr_auto_detect(temp_dir, zarr_format): except UnsupportedZarrFormat as exc: _skip_if_unsupported(exc, zarr_format) - result = runner.invoke(app, ["info", str(store_path)]) + result = runner.invoke(app, ["view", str(store_path)]) output = result.stdout + (result.stderr or "") assert result.exit_code == 0, output assert "5 × 4" in output diff --git a/uv.lock b/uv.lock index 7884068..0e90f12 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,88 @@ version = 1 revision = 3 requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] + +[[package]] +name = "adata-cli" +version = "0.4.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "h5py" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "rich" }, + { name = "typer" }, + { name = "zarr" }, +] + +[package.optional-dependencies] +dev = [ + { name = "anndata" }, + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "anndata", marker = "extra == 'dev'", specifier = ">=0.13" }, + { name = "h5py", specifier = ">=3.15.1" }, + { name = "numpy", specifier = ">=2.3.5" }, + { name = "pillow", specifier = ">=12.1.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.0.0" }, + { name = "rich", specifier = ">=14.2.0" }, + { name = "typer", specifier = ">=0.20.0" }, + { name = "zarr", specifier = ">=3.1.5" }, +] +provides-extras = ["dev"] + +[[package]] +name = "anndata" +version = "0.13.3.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "array-api-compat" }, + { name = "h5py" }, + { name = "legacy-api-wrap" }, + { name = "natsort" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "scipy" }, + { name = "scverse-misc", extra = ["settings"] }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, + { name = "zarr" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/54/11f25da4a78ef63bda43e9a6af7897995d1194e0650a940d96229d46be02/anndata-0.13.3.post0.tar.gz", hash = "sha256:9c564b4f04af7f84e45f28de9c4ee3455e989cb05ba1d800054033420f24e8b0", size = 2286709, upload-time = "2026-08-27T09:40:31.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/ad/33a5291066c4d51be58a8c537545402605feaa9fca49ff51ada6912d0234/anndata-0.13.3.post0-py3-none-any.whl", hash = "sha256:9e5eab0abadff2e97e9ea913d7f234c50bb6bea14e27acf7a5ecf14373e7e18b", size = 188750, upload-time = "2026-08-27T09:40:29.445Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "array-api-compat" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/12/65d297d3e425cb7fe5247b7ec14b4ed83539e9b5e89b65c15ea7ee274182/array_api_compat-1.15.0.tar.gz", hash = "sha256:53c5f922491bf15f62847afafc4e39eedfae57d218988fefb8cce39c2a9b3dea", size = 129305, upload-time = "2026-06-07T20:53:24.964Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/16/1a8fd2b19544b84575cf84ef7aa3ad4c173b756d5f087c91f85d1b295777/array_api_compat-1.15.0-py3-none-any.whl", hash = "sha256:7b1b9c53269061403fd5f45a8de349f16e7887653328bfa0c5f2d45299ff0a8e", size = 79113, upload-time = "2026-06-07T20:53:23.621Z" }, +] [[package]] name = "click" @@ -132,38 +214,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, ] -[[package]] -name = "h5ad" -version = "0.3.1" -source = { editable = "." } -dependencies = [ - { name = "h5py" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "rich" }, - { name = "typer" }, - { name = "zarr" }, -] - -[package.optional-dependencies] -dev = [ - { name = "pytest" }, - { name = "pytest-cov" }, -] - -[package.metadata] -requires-dist = [ - { name = "h5py", specifier = ">=3.15.1" }, - { name = "numpy", specifier = ">=2.3.5" }, - { name = "pillow", specifier = ">=12.1.0" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.4" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.0.0" }, - { name = "rich", specifier = ">=14.2.0" }, - { name = "typer", specifier = ">=0.20.0" }, - { name = "zarr", specifier = ">=3.1.5" }, -] -provides-extras = ["dev"] - [[package]] name = "h5py" version = "3.15.1" @@ -208,6 +258,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "legacy-api-wrap" +version = "1.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/49/f06f94048c8974205730d40beca879e43b6eee08efb0101cfb8623e60f41/legacy_api_wrap-1.5.tar.gz", hash = "sha256:b41ba6532f3ebfe3a897a35a7f97dec3be04b92a450f6c2bcf89f1b91c9cadf2", size = 11610, upload-time = "2025-11-03T13:21:12.437Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/5b/058db09c45ba58a7321bdf2294cae651b37d6fec68117265af90cde043b0/legacy_api_wrap-1.5-py3-none-any.whl", hash = "sha256:5a8ea50e3e3bcbcdec3447b77034fd0d32cb2cf4089db799238708e4d7e0098d", size = 10182, upload-time = "2025-11-03T13:21:11.102Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -229,6 +288,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "natsort" +version = "8.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e2/a9/a0c57aee75f77794adaf35322f8b6404cbd0f89ad45c87197a937764b7d0/natsort-8.4.0.tar.gz", hash = "sha256:45312c4a0e5507593da193dedd04abb1469253b601ecaf63445ad80f0a1ea581", size = 76575, upload-time = "2023-06-20T04:17:19.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/82/7a9d0550484a62c6da82858ee9419f3dd1ccc9aa1c26a1e43da3ecd20b0d/natsort-8.4.0-py3-none-any.whl", hash = "sha256:4732914fb471f56b5cce04d7bae6f164a592c7712e1c85f9ef585e197299521c", size = 38268, upload-time = "2023-06-20T04:17:17.522Z" }, +] + [[package]] name = "numcodecs" version = "0.16.5" @@ -326,6 +394,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + [[package]] name = "pillow" version = "12.1.0" @@ -404,6 +518,110 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pydantic" +version = "2.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3f/76358795aa7a8c6d4f36e2cb828ad1c90ee118e1393a9281664f5aade9d4/pydantic_core-2.46.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d", size = 2076516, upload-time = "2026-08-28T09:58:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/db/50/26b091836076ce4cb2fac264186936acc069e0595772cfd02a563bc4761a/pydantic_core-2.46.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e", size = 1922874, upload-time = "2026-08-28T09:58:23.766Z" }, + { url = "https://files.pythonhosted.org/packages/09/f0/2a8ce3849e299d44e2d2c196b6082643a3235565a735cb51db7a6261f614/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29", size = 1951772, upload-time = "2026-08-28T09:58:25.435Z" }, + { url = "https://files.pythonhosted.org/packages/87/46/ac0dc8bdd9e6048183a14eb127764e7ad9240021c17513074a4711b0e31e/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4", size = 2031832, upload-time = "2026-08-28T09:58:27.102Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/339de5bef7be36301a2231eaa52e62163742c2281f11b5f4892bc79785cd/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a", size = 2208645, upload-time = "2026-08-28T09:58:28.948Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a0/9ff22b797724262da14427abaed4dd1d864a139693fc5e7809114376a716/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62", size = 2265935, upload-time = "2026-08-28T09:58:30.625Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2", size = 2066284, upload-time = "2026-08-28T09:58:32.289Z" }, + { url = "https://files.pythonhosted.org/packages/c0/02/7f6156ffc926857f1c37c07d9a388682865a81830ab6a1b637082c25e399/pydantic_core-2.46.5-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869", size = 2105889, upload-time = "2026-08-28T09:58:33.986Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/e781d357ebe09fc929f995700f1b3503e8897f1cece183ecb1300d4d67e9/pydantic_core-2.46.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5", size = 2158006, upload-time = "2026-08-28T09:58:35.647Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/644597d84ab400e50609c192120b85c9681c22d3a20461b9060a79be0a7a/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3", size = 2158408, upload-time = "2026-08-28T09:58:37.38Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ee/ca3b7b3a4b3769ffe9ce9432a7c9be755de9593a46d3b0d54d0409323e44/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b", size = 2309609, upload-time = "2026-08-28T09:58:39.22Z" }, + { url = "https://files.pythonhosted.org/packages/ce/52/39fa1f451486019524ca685020390e7ca351832fd874530ba30c8628e6dc/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0", size = 2342618, upload-time = "2026-08-28T09:58:40.89Z" }, + { url = "https://files.pythonhosted.org/packages/81/5e/468fc630568c61dcef3cd47ad32ffbeed9af643f49208d1ea86ab4f890c4/pydantic_core-2.46.5-cp312-cp312-win32.whl", hash = "sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b", size = 1939475, upload-time = "2026-08-28T09:58:42.591Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl", hash = "sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8", size = 2043140, upload-time = "2026-08-28T09:58:44.272Z" }, + { url = "https://files.pythonhosted.org/packages/af/dd/0c1a050299147c746e5256db16d645ab5efd4f78c59937d581a0524e74a2/pydantic_core-2.46.5-cp312-cp312-win_arm64.whl", hash = "sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084", size = 1997729, upload-time = "2026-08-28T09:58:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/f5/37/5abe39a8372a61d3dc3c1338fc504281c01b32fdb3169cd7187153b56d3e/pydantic_core-2.46.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0", size = 2075885, upload-time = "2026-08-28T09:58:47.856Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/6323b1f8b217780454c61304bcd2b38ae4762f50754414124603ccc90bb2/pydantic_core-2.46.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff", size = 1922768, upload-time = "2026-08-28T09:58:49.58Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a3/c05ca796e1197618a774b01e596aeedfefc2f7d8c01ae3054e910b120e8a/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931", size = 1951241, upload-time = "2026-08-28T09:58:51.511Z" }, + { url = "https://files.pythonhosted.org/packages/68/32/33bc39ac705c52cffc908e8389f9754fdb208aea5c69cceddf4eb3ce99af/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f", size = 2031975, upload-time = "2026-08-28T09:58:53.166Z" }, + { url = "https://files.pythonhosted.org/packages/b0/70/2333e885c0f6a67bc105c5916965dac9b57f2718ee20d81d1a06a4ebdc13/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038", size = 2208542, upload-time = "2026-08-28T09:58:55.017Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ea/296debfb4264207bbda5936133892e027c0a58875ad53ebd512fba8ec3a2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f", size = 2264692, upload-time = "2026-08-28T09:58:56.767Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1", size = 2066633, upload-time = "2026-08-28T09:58:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/8d/db/f9e9d0c97445987b2084823d5c240de88087338f04fc2cfaa2df186b8049/pydantic_core-2.46.5-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761", size = 2105235, upload-time = "2026-08-28T09:59:00.421Z" }, + { url = "https://files.pythonhosted.org/packages/07/c5/79169b047b3b2c3e99e04bc76372af9637e0bf6db638274fa927df96369e/pydantic_core-2.46.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5", size = 2157367, upload-time = "2026-08-28T09:59:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/26/b5/ba6057afb7c291bd449f51b867f95aef2072941c4ce4e5c31d6ffd132d3b/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e", size = 2158420, upload-time = "2026-08-28T09:59:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/2057abecaafdc22912afa819603a51f0a62d40643b7c4871c51721fea9be/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed", size = 2309588, upload-time = "2026-08-28T09:59:06.048Z" }, + { url = "https://files.pythonhosted.org/packages/71/9d/881156dc404e27479c4246128d73538464cab4a239bec61995e227644c30/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519", size = 2341866, upload-time = "2026-08-28T09:59:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/5a/38/d66f443a259f84d13babdceae568e572b0ed26da17ca5d0a649ebb110a67/pydantic_core-2.46.5-cp313-cp313-win32.whl", hash = "sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea", size = 1938580, upload-time = "2026-08-28T09:59:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl", hash = "sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5", size = 2041980, upload-time = "2026-08-28T09:59:12.396Z" }, + { url = "https://files.pythonhosted.org/packages/5a/48/4222d90b1c67568bace4dec6dca6271449c66de3595d72b6d098f5fde597/pydantic_core-2.46.5-cp313-cp313-win_arm64.whl", hash = "sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575", size = 1997213, upload-time = "2026-08-28T09:59:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" }, + { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" }, + { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" }, + { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" }, + { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" }, + { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" }, + { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" }, + { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" }, + { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" }, + { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" }, + { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" }, + { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" }, + { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" }, + { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" }, + { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" }, + { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" }, + { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" }, + { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" }, + { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" }, + { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" }, + { url = "https://files.pythonhosted.org/packages/df/dd/053c2e4303f791f3b8f8a14ab0b22008e8eb21d868c0c90b4f9be705b76a/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942", size = 2062540, upload-time = "2026-08-28T10:01:00.318Z" }, + { url = "https://files.pythonhosted.org/packages/d7/dd/a18df751a5e37dd51bfad7f68e766999125bebe68c9e1d10a493ad01bd63/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f", size = 1902040, upload-time = "2026-08-28T10:01:02.529Z" }, + { url = "https://files.pythonhosted.org/packages/b7/13/01d40f9d07ce8a779fd6e0bd8ad4fba91309500dd67b869e2e219d261a6d/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433", size = 1967479, upload-time = "2026-08-28T10:01:05.004Z" }, + { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -443,6 +661,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -502,6 +741,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/2a/a1810c8627b9ec8c57ec5ec325d306701ae7be50235e8fd81266e002a3cc/rich-14.3.1-py3-none-any.whl", hash = "sha256:da750b1aebbff0b372557426fb3f35ba56de8ef954b3190315eb64076d6fb54e", size = 309952, upload-time = "2026-01-24T21:40:42.969Z" }, ] +[[package]] +name = "scipy" +version = "1.18.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/f7/240c110c08693826b4513a52f5717d62ec7c7af72f2920821247c03b17b3/scipy-1.18.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:457fd7a2a8edeb044ab6ffbc0aa03ff6cd18491356e5e0c834d76ce621b916d1", size = 31111061, upload-time = "2026-08-21T23:23:44.522Z" }, + { url = "https://files.pythonhosted.org/packages/05/4a/78c6285577c375e7cf27277ea8ee6961224327f1e1a0c44af5f17f23635c/scipy-1.18.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:e708533e8b2ae2497d65346538a7dcc92814410b25b81432eac66de0f2af8265", size = 28733332, upload-time = "2026-08-21T23:23:50.015Z" }, + { url = "https://files.pythonhosted.org/packages/a5/f6/a5b82f8abbe14d134691b8b903696f701d25a081353a29dc655c364d9e62/scipy-1.18.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:7bbf207c4453ce1ad2e00b17313852b33310b83090c2311bdaf97f93c0380d12", size = 20475078, upload-time = "2026-08-21T23:23:54.138Z" }, + { url = "https://files.pythonhosted.org/packages/23/22/0858a0bbd6b3e825ceb8cd9baf9eaf3b2f2b1d77727eb6be40500bcdc92f/scipy-1.18.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:78c0665edead396b1abb4897c41a5c1d9bf090c8a637a4c20a61678e0a264e66", size = 23108904, upload-time = "2026-08-21T23:23:57.824Z" }, + { url = "https://files.pythonhosted.org/packages/75/9a/2e71719f31eaefe0e3a1706c4a1ded94e664bfd95ffca2b219a671faee01/scipy-1.18.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c085faa2cfa879c5141df483f836f4d691045a078224a670fa570fa01612d89", size = 34025113, upload-time = "2026-08-21T23:24:02.209Z" }, + { url = "https://files.pythonhosted.org/packages/df/64/ff35eb9e54894cf471ff4716abd3c81eb0a0626869217ce3e6ba4ccf17d7/scipy-1.18.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218", size = 35344199, upload-time = "2026-08-21T23:24:07.844Z" }, + { url = "https://files.pythonhosted.org/packages/d3/af/c5538be1792f7034c12c7db6ee67cace58253c7b87b122d68253eaf5de89/scipy-1.18.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c35d74ce0e193ff740c2f2be2ac913ddc232fe6c1ff40b26cfecb9c670c63314", size = 35639587, upload-time = "2026-08-21T23:24:13.05Z" }, + { url = "https://files.pythonhosted.org/packages/91/4c/075e4f66471bac101141ac739e9e135549be1bae584571bd03a530c056e1/scipy-1.18.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1", size = 37480330, upload-time = "2026-08-21T23:24:19.608Z" }, + { url = "https://files.pythonhosted.org/packages/39/e7/979fd14e75008623df31ba70d6bb144700f68feadcea042021c06a05bf82/scipy-1.18.1-cp312-cp312-win_amd64.whl", hash = "sha256:5e4d44984abc0020154ea81b247adeddcc3ac5527b975ff798bd1ba0adc513c2", size = 36658278, upload-time = "2026-08-21T23:24:25.463Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0b/e1525354ff9d7d5feb6d1b31af6d14072e5c91e9607b421fa1ec889660b3/scipy-1.18.1-cp312-cp312-win_arm64.whl", hash = "sha256:d65d448389b8436493abcf629cc94ad0cf32aecaf06e1acca1de53cc795f2f12", size = 24400588, upload-time = "2026-08-21T23:24:30.579Z" }, + { url = "https://files.pythonhosted.org/packages/b6/55/4540ee0f9c42a9ad7109d0d1a8cc70de54c3572b01c6693a2b1c70e90ceb/scipy-1.18.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:3ab3523da44749156e1f68b464dc56af11ae4cbc5c739a49d05f32b982eca9f3", size = 31089958, upload-time = "2026-08-21T23:24:35.8Z" }, + { url = "https://files.pythonhosted.org/packages/2a/f5/769f36d14922b8071a43e95d24d18b6bdafad10d7f5cf647867e1ac052bc/scipy-1.18.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6fb6a55cc0ba97b59a1f288fb86dc6fce8bdfc0fffcbfd015e3a954bf2a2d93", size = 28715106, upload-time = "2026-08-21T23:24:40.775Z" }, + { url = "https://files.pythonhosted.org/packages/9a/d7/21d890274f75ea37a8209d5519e72da3da90302e3b9fb8397a0918386a62/scipy-1.18.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ea324d9dd34c38bfb9bec8ca4d1b407db97dbb74029f566b8e322b1b6fe56fe6", size = 20456846, upload-time = "2026-08-21T23:24:45.066Z" }, + { url = "https://files.pythonhosted.org/packages/ec/01/798430ecea2e78ec7c02663d5f71c007bb6abeca931080debd40d7fa55ea/scipy-1.18.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:75b00eb8fb802090aa903f4ea1c7f5a584779f967361e68b7e98e531cc2d7174", size = 23087986, upload-time = "2026-08-21T23:24:49.539Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5f/4634e9d35c68496e4e34cb6946eafab044458e6cedab42b40b6588e475b6/scipy-1.18.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d416b16cccfd70fbf62400e84d0bb2f4e6af519a45557f1692c749b37f14b315", size = 33998146, upload-time = "2026-08-21T23:24:54.714Z" }, + { url = "https://files.pythonhosted.org/packages/41/48/6450ed9243315322bbc19ac57b9b70d66a20bf1d38d124c96bc4bf6af9ea/scipy-1.18.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9", size = 35312578, upload-time = "2026-08-21T23:25:00.44Z" }, + { url = "https://files.pythonhosted.org/packages/00/bd/bf5a4be6a3525676499f6dff307991739ff6fdcad1481b1aeb6745339f58/scipy-1.18.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c825cef2f49e46753726a7181a8e199804a912b29519ada542c6ebc654951899", size = 35612621, upload-time = "2026-08-21T23:25:06.144Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4e/3c45c33e00a77996c4b1cb707929f833ba7b1d522ee29f882512c330676d/scipy-1.18.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07", size = 37457323, upload-time = "2026-08-21T23:25:12.483Z" }, + { url = "https://files.pythonhosted.org/packages/93/0e/e0348fbc0dbab65c114cf78957e7dfeb49f8e8b556b4d930cc12ff195e18/scipy-1.18.1-cp313-cp313-win_amd64.whl", hash = "sha256:559ed65f60c1af5a03f3912605a1b5114f522c7c32fb23c3376ae8f03219fe28", size = 36622841, upload-time = "2026-08-21T23:25:18.722Z" }, + { url = "https://files.pythonhosted.org/packages/50/a8/6a77f5f267c555108f0a864b6db714363dab567a8266422a79a385f9232b/scipy-1.18.1-cp313-cp313-win_arm64.whl", hash = "sha256:cd479fc04dd9401e3b4f49e76518768ef99c4f517a98c284eb091fd725719adf", size = 24399315, upload-time = "2026-08-21T23:25:23.458Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/d8eb4e280ddb56a4ab2c6f02ee49b56b23f6e977cf0802fd6d68dbef14f5/scipy-1.18.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:83de5453a7799afc9048b4616bd085cef126e36412f0ea2f6370c36a2a3a51e7", size = 31090936, upload-time = "2026-08-21T23:25:28.686Z" }, + { url = "https://files.pythonhosted.org/packages/2a/49/59ea385dc3a62ff498ddf3cfff7c2b41b0f9f9d3c4122b3f1dcb6d6327fe/scipy-1.18.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:9554bcc6d715ee87a633a3cc8e7703c6628b100dd29cb8a2efc4c0533c7ff729", size = 28725221, upload-time = "2026-08-21T23:25:33.244Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/6b0c288c50942d78193696c9f15f9a0874f5178aa0ddf40f83d9924b3e8d/scipy-1.18.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc", size = 20466839, upload-time = "2026-08-21T23:25:37.516Z" }, + { url = "https://files.pythonhosted.org/packages/4b/e0/54fd3793c729e3b936782f181b59cbb1205bf250ab605a16cb1ba61cdd5e/scipy-1.18.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:88f0e784020649f88ea48c9f5ddfa403bf9205820667c0914740b392035afb82", size = 23089121, upload-time = "2026-08-21T23:25:42.019Z" }, + { url = "https://files.pythonhosted.org/packages/0b/56/030af62bea3cf878e0028515dff78c123b01633606a879b63f42d2db99cc/scipy-1.18.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d3ab0e8c69a17dd3559eab8cbb88f258e285c94d572c2719033f90f83290c89", size = 34053851, upload-time = "2026-08-21T23:25:47.998Z" }, + { url = "https://files.pythonhosted.org/packages/6b/89/2a844506d49651e9aa1af6ef95b6bd8031cb1d5a4375edec6155037e04cf/scipy-1.18.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac0333bdf38309aa3dcbe7e3fa7ea29e7a2c37c6ea306a757b700ded8e4596ad", size = 35329183, upload-time = "2026-08-21T23:25:53.522Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/c7370c3640e92ac9613cbf26cb3f729f9b12ddf1727b55b94b53b24d6f48/scipy-1.18.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:911de823097db8b63f034299d12662db93344e6ffa0b881cbb57748974b70168", size = 35672551, upload-time = "2026-08-21T23:25:59.387Z" }, + { url = "https://files.pythonhosted.org/packages/24/16/ec8536f351421f8bf60a1120930638f83790f4710b8230446aca3d6159d4/scipy-1.18.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:95298364e251be3e60249facbeeca03631d3bb7584f85879516ec55ac717b81f", size = 37469416, upload-time = "2026-08-21T23:26:05.432Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/d73da0d28f16c45bb9b0a5691b91610b0275c5ef0eb5e43c87cf2dc1bf31/scipy-1.18.1-cp314-cp314-win_amd64.whl", hash = "sha256:78a0d7c918e74a232394117160e7e3db503377572a45bcef8826e4ab8a35feba", size = 37362755, upload-time = "2026-08-21T23:26:11.366Z" }, + { url = "https://files.pythonhosted.org/packages/89/25/e996e4dc74e10e227b1e14db5eaf6608bb6dd33884a64851c38f18dd4249/scipy-1.18.1-cp314-cp314-win_arm64.whl", hash = "sha256:cbf38d043c1aa4ab306e1ada6ab6eddacc3322a20b7af1b30bc93254b366fe09", size = 25036090, upload-time = "2026-08-21T23:26:15.887Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c9/c00213f92309d753b48903e6a451b87eb52ff5b7a16e789d1568bbf221c4/scipy-1.18.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0fcb3c93519f27bb4f0c4b0f7802cdcaca7fcf93267b75edda2e9f4e8a55cbd7", size = 31485550, upload-time = "2026-08-21T23:26:20.776Z" }, + { url = "https://files.pythonhosted.org/packages/74/b2/e3067c487982d4eeab2938928529410370c06fea84a4d3f4925e7d96647d/scipy-1.18.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ddef79fb382df40104a19bb7151b3b23e57c1778fcf857c71ceecd9bd264513f", size = 29174642, upload-time = "2026-08-21T23:26:25.395Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ab/374c9fe2d1ec014e576c781a4b5d8e1ba340e8f6b4638c16f711d2b194f0/scipy-1.18.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:0e82073ecc7acc6436fac4b31674109c7e1d3e596789767eda01258a8c9e8123", size = 20916357, upload-time = "2026-08-21T23:26:30.112Z" }, + { url = "https://files.pythonhosted.org/packages/90/38/223915c88a17317cafbf8ca2a42b11c265a9fb1e804aa665544132b5fe8a/scipy-1.18.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:8bcf3c1ba5d6456e2effd30fcbd3459b044d683fcdac79a2e6830f0bdf7de487", size = 23482611, upload-time = "2026-08-21T23:26:34.846Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d1/db0948da8ca57a80b36520ef0a768b967d99f3af65f4b6f1bf6362ad4dd4/scipy-1.18.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cfbf154f2ba187f2ed6cce2639efff7d105f1140573642c0161615b6d91d6a87", size = 34143202, upload-time = "2026-08-21T23:26:40.4Z" }, + { url = "https://files.pythonhosted.org/packages/87/53/39d046cc7574ed6acacb6bd5723e220107ece80bff12faaf3efc4ddeede4/scipy-1.18.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1d33a7836f7ddc1993427966a0823468ec41bcbdb1a9f9942d1d7e57f803ba3", size = 35380876, upload-time = "2026-08-21T23:26:46.1Z" }, + { url = "https://files.pythonhosted.org/packages/f9/da/32e0e799d875a85ca57d9bde6c78148afcc0e38276df683d95854eadc8c3/scipy-1.18.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f4b8bc363b6d65ee2152bec57568e3c52639bb34c46057b09857a307ed5e21d", size = 35770885, upload-time = "2026-08-21T23:26:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/88/2e/f97a666d362fee68b18f41c9c30ed502ca5c98b549749bfcb52a8b74d1eb/scipy-1.18.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11c423f1049c5755ad4409af52a9ada1cff96fe9b50795d4af3619f292901239", size = 37525424, upload-time = "2026-08-21T23:26:56.751Z" }, + { url = "https://files.pythonhosted.org/packages/ca/d5/a9e765a84654ebba8479a1fd1b059ced1af72b168a3b2a3a46540ea38d20/scipy-1.18.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c24acac1e18912761c4700239bbc1fd32f615af690f1584d49b35859be51324d", size = 37416961, upload-time = "2026-08-21T23:27:01.546Z" }, + { url = "https://files.pythonhosted.org/packages/ee/16/e79e0d1c63ef698879d85439d37e9fb434e3b804e506a6991038d086ebd9/scipy-1.18.1-cp314-cp314t-win_arm64.whl", hash = "sha256:9f2897bf7737392ad0d5213ea7b6add72a4edf5679b3153106aeb88b6507b3b9", size = 25331848, upload-time = "2026-08-21T23:27:05.884Z" }, + { url = "https://files.pythonhosted.org/packages/be/4f/1bd37c883b67163e2ca1f60977a399500e6879c15defecac62831c8d078d/scipy-1.18.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:eb0dfcf4e28a99c12c999744a2ff67c9b06200e20401c7c88186e33552a46331", size = 31091484, upload-time = "2026-08-21T23:27:11.051Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c5/ba929d7feb9b2332f96827c12e0e924b61973b59b4dea383b603372c65ce/scipy-1.18.1-cp315-cp315-macosx_12_0_arm64.whl", hash = "sha256:30f464bee641fa8e282577c7dce027308403213c6ca8270bba73285c91024bc5", size = 28725057, upload-time = "2026-08-21T23:27:15.9Z" }, + { url = "https://files.pythonhosted.org/packages/a4/19/68f1c50f609d955d230e66d25d02bd3e1e167ec540232135354fb9a4b9e3/scipy-1.18.1-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:1bca3b943fc2567ea49cd02c99abde49da4d5178ec46f624bd8255cda8755beb", size = 20466734, upload-time = "2026-08-21T23:27:20.044Z" }, + { url = "https://files.pythonhosted.org/packages/ef/6d/319fa29b73d1802fa80b32a6eaf3f5be456ef81526da2716a9493bcb5501/scipy-1.18.1-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:c9d18a33309122074ea483dd92dd444189166b8b2ec429fe9ed5ac73c7a0aa23", size = 23089664, upload-time = "2026-08-21T23:27:24.345Z" }, + { url = "https://files.pythonhosted.org/packages/b7/db/30992f9b51a63de671daf3888ffd18378b6cb9ec9f2c972264238ffa7fd6/scipy-1.18.1-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82f201b4c878551d48558337aab270d3c6cca5507b8737c8d8a608d234cccde0", size = 34054035, upload-time = "2026-08-21T23:27:29.409Z" }, + { url = "https://files.pythonhosted.org/packages/91/d4/bf3e735dc0b9d5a8ff45079d2540e17d3aff7a2f0048dd8f552ffd031d2b/scipy-1.18.1-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5", size = 35333883, upload-time = "2026-08-21T23:27:34.293Z" }, + { url = "https://files.pythonhosted.org/packages/19/93/12d78ce9f871fe945fca588d32644e6e63f553c2a35c564d73f3b22a3313/scipy-1.18.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ceb30a00ce7c92d459819443d29ca486d882b83fb6738bdcbb2a1cce94ac5daa", size = 35673124, upload-time = "2026-08-21T23:27:39.059Z" }, + { url = "https://files.pythonhosted.org/packages/70/cd/886219313a1012a48e6ae0ec4f302c837151beb92e1ff0d709ef8fdfc488/scipy-1.18.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f29633129f9fa7e88a3f0fca835de2d030bfc9643f7799e1a0c46cee24d38fc7", size = 37470753, upload-time = "2026-08-21T23:27:44.435Z" }, + { url = "https://files.pythonhosted.org/packages/17/6c/a776888ce618bee54fbde26172f0f46ac1da70d27b63861797fe78e1904b/scipy-1.18.1-cp315-cp315-win_amd64.whl", hash = "sha256:92c14f5bdbfb6216315ce33e78080474082de8b3830122ba97809bfbe65f75c0", size = 37361483, upload-time = "2026-08-21T23:27:49.334Z" }, + { url = "https://files.pythonhosted.org/packages/ab/09/97b651691322ebee97999b017ffc18a15a0b815103844c97e8da9d469731/scipy-1.18.1-cp315-cp315-win_arm64.whl", hash = "sha256:e402cf31eb68f453dbb2d36fc6d722b33f24a55d68b2ae1d92fa6305ca71c298", size = 25035883, upload-time = "2026-08-21T23:27:53.596Z" }, + { url = "https://files.pythonhosted.org/packages/ed/0f/9ec20467bbabd0d44e2a77d0fd3d124f884b4d67df92af82c91d2d6a486f/scipy-1.18.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:2a0b02f9fc46f8520330c23d45e6560db7e3a0d927232139427637f98943e11d", size = 31474926, upload-time = "2026-08-21T23:27:57.993Z" }, + { url = "https://files.pythonhosted.org/packages/8a/58/dcb79161e56efbedc50079fcd2f5fe427a0ebb53022eb476aa73c015ad8f/scipy-1.18.1-cp315-cp315t-macosx_12_0_arm64.whl", hash = "sha256:1d73131e358976663dd969e1fb4ed1404b815cd977eaaedc3b3a133ba2d81c35", size = 29164940, upload-time = "2026-08-21T23:28:03.062Z" }, + { url = "https://files.pythonhosted.org/packages/71/d3/1eeea80c817fcb8ef7bd4a05a58824977a0e57a375cfc3d7ea7c911c01ad/scipy-1.18.1-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:bff0b729edd992766136b34e39cc76bc2fad905aa58897ee72a9cd000a6d8443", size = 20906742, upload-time = "2026-08-21T23:28:07.642Z" }, + { url = "https://files.pythonhosted.org/packages/54/46/e59350428b6099301a20128108c995e2eb175a43f383af9a346e38824f9b/scipy-1.18.1-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:10ac20c69d880f77f375db44c22e3e6a644f9fefa291d4cd2fb9790a89fc99fd", size = 23472183, upload-time = "2026-08-21T23:28:12.109Z" }, + { url = "https://files.pythonhosted.org/packages/89/31/cc91623fa98f0621766a0f0aaaadb2c66de74a7ea7e3837164f6e4354260/scipy-1.18.1-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33a834464fdabc0f26a45508df31b3cc5d028e04dbf6c5ed398541418e0a12fe", size = 34130796, upload-time = "2026-08-21T23:28:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/fc/3e/8572ef536957ddb8aa81bb4090d9e25f257e3b4e05d97deb54319deb8a3a/scipy-1.18.1-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49023963c193dacee096301452f223ee24d86ec5807f8df93c0f7221d119e305", size = 35374253, upload-time = "2026-08-21T23:28:23.732Z" }, + { url = "https://files.pythonhosted.org/packages/b5/c6/59fdeffb4f1435299f93d9dc8140b43ad2916e6cfc944be6c3041fcec86d/scipy-1.18.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:d84a09d0dad90ba6525d8ac1c2334b33e64bf3ccfe9e841f02feb867a22681e4", size = 35758543, upload-time = "2026-08-21T23:28:29.431Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d9/135be205d9de8783193aff9cc3bf483a03a38e4b29432c954e8cb66ac14e/scipy-1.18.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:179ce34a8d0fe273d8883ba59e17e052247d08973dfcb743ca52bb1cce2d60b0", size = 37521946, upload-time = "2026-08-21T23:28:35.245Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a2/5b7d5270621ab7cfa3f7766067bf95dc360b5efb6394694e8143b4156e2b/scipy-1.18.1-cp315-cp315t-win_amd64.whl", hash = "sha256:5632e3ae3d09197c446310cd5187de63e28448ce22f0f67b2b93d97503c0c230", size = 37408295, upload-time = "2026-08-21T23:28:40.724Z" }, + { url = "https://files.pythonhosted.org/packages/63/ad/741c19fcb66755ff953daf9243af8480e4bf3d7fbe57583c178c7d2b6b51/scipy-1.18.1-cp315-cp315t-win_arm64.whl", hash = "sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a", size = 25319710, upload-time = "2026-08-21T23:28:45.713Z" }, +] + +[[package]] +name = "scverse-misc" +version = "0.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "session-info2" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/0a/1f7c48d1a1fba2fac8e4b49eee1e532d35f224e621502a13052f831d1f01/scverse_misc-0.1.6.tar.gz", hash = "sha256:2310c2a0d2159311f1b44759cbd2f83b543362556396d93cb70cff01647ec835", size = 52968, upload-time = "2026-09-11T07:30:58.945Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/16/8e225fd30ae5006a7f425803d33f69b800668b70d5395ecb63a17f4524b7/scverse_misc-0.1.6-py3-none-any.whl", hash = "sha256:5fc1d262981d100f9936c0729f2c38f13e8284d070c56613a4d07d58dd29c68a", size = 28763, upload-time = "2026-09-11T07:30:57.67Z" }, +] + +[package.optional-dependencies] +settings = [ + { name = "pydantic-settings" }, + { name = "python-dotenv" }, +] + +[[package]] +name = "session-info2" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/34/b13340d5b4793ad5cd80118e0ebd484ac70250069110746da8026c34768c/session_info2-0.4.2.tar.gz", hash = "sha256:d85d730621d6f75df60e15dad21258f2750f04c0c0ee5958a7bf92342c039d76", size = 25385, upload-time = "2026-08-04T09:02:04.087Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/41/b5af3c12d874fcbdb45abd67500aff8030de0d350a5d6163da45378609c4/session_info2-0.4.2-py3-none-any.whl", hash = "sha256:dbf5f58769115651fad28e8ffc930b6a848a629a242ce2ecd730d12f5e3e9315", size = 17665, upload-time = "2026-08-04T09:02:02.9Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -511,6 +849,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "typer" version = "0.21.1" @@ -528,11 +875,32 @@ wheels = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/31/3d74fa778a63b98b7374323befcc0be5ab3bd94afd4096a0124e7379152c/tzdata-2026.4.tar.gz", hash = "sha256:f1b8bd365d8d210c55353f4d7f8d6d8561c0ba50d704b700d195a9424bba0d79", size = 199350, upload-time = "2026-09-12T12:56:03.251Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl", hash = "sha256:c2169a8b0a7a5e9674da5a135ccdfb2b3e671b333ed9fed17b41f73c34476e81", size = 347494, upload-time = "2026-09-12T12:56:01.67Z" }, ] [[package]] From f13046eaace993df0f1aa03575b782a8ad8d0463 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 12:21:49 +0100 Subject: [PATCH 11/23] Add create, split, concat and query-based subsetting Four commands, all built on the elements layer added in the previous change rather than each reimplementing the format. `create` writes a valid empty store -- root attributes, obs/var frames with real indices, tagged mapping groups -- so `import` has something to fill in. `import` loses its obs/var restriction and gains an `image` subcommand, so a whole object can be assembled from the command line. `split` writes one store per distinct value of an annotation column, with label sanitisation, collision suffixes and a CSV manifest following the conventions of cellgeni/scraft's split_h5ad, but streaming rather than reading the source into memory. Closes #2. `subset` gains --obs-query/--var-query over a small predicate language (==, !=, <, <=, >, >=, in, not in, and, or, not, parentheses) evaluated chunk-wise. Only the columns a query mentions are read. This deliberately is not SQL: anything more involved is better served by exporting to CSV and using duckdb. A var query propagates to raw/ by name, so raw stays consistent. `concat` joins along obs with inner/outer alignment, --label/--keys/ --index-unique, and merge strategies for var and uns. Verified against anndata.concat: var order, obs order and X values agree exactly for both joins. obs columns keep their dtypes -- categoricals union their category sets and remap codes, nullable columns keep their masks, and a column missing from one input is padded rather than dropped. obsp/varp and raw are not carried over, as in anndata's own implementation. Co-Authored-By: Claude Opus 5 --- README.md | 26 +- src/adata/cli.py | 372 ++++++++++++- src/adata/commands/__init__.py | 3 + src/adata/commands/concat.py | 21 + src/adata/commands/create.py | 93 ++++ src/adata/commands/import_data.py | 15 + src/adata/commands/split.py | 147 +++++ src/adata/core/concat.py | 858 ++++++++++++++++++++++++++++++ src/adata/core/query.py | 268 ++++++++++ src/adata/core/select.py | 122 +++++ src/adata/core/subset.py | 113 ++-- src/adata/formats/dataframe.py | 26 +- src/adata/formats/image.py | 26 + tests/test_cli.py | 2 +- tests/test_commands_phase2.py | 452 ++++++++++++++++ tests/test_import.py | 11 +- tests/test_query.py | 83 +++ 17 files changed, 2566 insertions(+), 72 deletions(-) create mode 100644 src/adata/commands/concat.py create mode 100644 src/adata/commands/create.py create mode 100644 src/adata/commands/split.py create mode 100644 src/adata/core/concat.py create mode 100644 src/adata/core/query.py create mode 100644 src/adata/core/select.py create mode 100644 tests/test_commands_phase2.py create mode 100644 tests/test_query.py diff --git a/README.md b/README.md index ab893ad..05a1f8e 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,31 @@ Run help at any level (e.g. `adata --help`, `adata export --help`). - `view` – AnnData-aware inspection: store layout, shapes, and encodings; supports drilling into paths like `obsm/X_pca` or `uns`. - `ls` – list the contents of any HDF5 or Zarr store as a tree, with no AnnData assumptions (works on `.loom` and plain `.h5`); `-1` emits bare paths for piping. -- `subset` – stream and write a filtered copy based on obs/var name lists, preserving dense and sparse matrix encodings. +- `create` – write a new, empty AnnData store for `import` to fill in. +- `subset` – stream and write a filtered copy, selected by obs/var name lists (`--obs`/`--var`) or by expression (`--obs-query`/`--var-query`). +- `split` – write one store per distinct value of an annotation column, with a CSV manifest. +- `concat` – concatenate stores along the obs axis, with `--join inner|outer` and merge strategies for var and uns. - `export` – extract data from a store; subcommands: `dataframe` (any dataframe group to CSV), `array` (dense to `.npy`), `sparse` (CSR/CSC to `.mtx`), `dict` (JSON), `image` (PNG). Results go to stdout when no `--output` is given. -- `import` – write new data into a store; subcommands: `dataframe` (CSV → obs/var), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON). +- `import` – write new data into a store at any path; subcommands: `dataframe` (CSV), `array` (`.npy`), `sparse` (`.mtx`), `dict` (JSON), `image` (PNG/JPEG/TIFF). + +### Building a store from scratch + +```bash +adata create out.h5ad --obs-names cells.txt --var-names genes.txt +adata import sparse out.h5ad X counts.mtx --inplace +adata import dataframe out.h5ad obs cells.csv --inplace -i cell_id +adata import array out.h5ad obsm/X_umap umap.npy --inplace +adata import dict out.h5ad uns/params params.json --inplace +``` + +### Filtering without a name list + +```bash +adata subset data.h5ad -o cortex.h5ad --obs-query "cluster == Cortex_2" +adata subset data.h5ad -o big.h5ad -q "n_counts > 1000 and cluster in A,B" +adata split data.h5ad --by sample -o per_sample/ +adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample +``` See [docs/GET_STARTED.md](docs/GET_STARTED.md) for a short tutorial. diff --git a/src/adata/cli.py b/src/adata/cli.py index 4b556fc..9da575f 100644 --- a/src/adata/cli.py +++ b/src/adata/cli.py @@ -7,6 +7,10 @@ import typer from adata.commands import ( + MERGE_STRATEGIES, + concat_stores, + create_store, + split_store, list_store, show_info, subset_h5ad, @@ -117,6 +121,75 @@ def info( view(file=file, entry=entry, tree=tree, depth=depth) +# ============================================================================ +# CREATE command +# ============================================================================ +@app.command("create") +def create( + output: Path = typer.Argument( + ..., + help="Path of the store to create (.h5ad or .zarr)", + dir_okay=True, + file_okay=True, + ), + n_obs: Optional[int] = typer.Option( + None, "--n-obs", help="Number of observations (cells)" + ), + n_var: Optional[int] = typer.Option( + None, "--n-var", help="Number of variables (genes)" + ), + obs_names: Optional[Path] = typer.Option( + None, + "--obs-names", + help="File of obs names, one per line (sets --n-obs)", + exists=True, + readable=True, + ), + var_names: Optional[Path] = typer.Option( + None, + "--var-names", + help="File of var names, one per line (sets --n-var)", + exists=True, + readable=True, + ), + zarr_format: Optional[int] = typer.Option( + None, "--zarr-format", help="Zarr spec version to write (2 or 3)" + ), + force: bool = typer.Option( + False, "--force", "-f", help="Overwrite an existing store" + ), +) -> None: + """ + Create a new, empty AnnData store. + + The result is a valid AnnData object straight away; fill it in with + `adata import`. Give each axis either a size or a file of names. + + Examples: + adata create out.h5ad --n-obs 5000 --n-var 2000 + adata create out.zarr --obs-names cells.txt --var-names genes.txt + adata import sparse out.h5ad X counts.mtx --inplace + """ + if zarr_format is not None and zarr_format not in (2, 3): + console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.") + raise typer.Exit(code=1) + + try: + create_store( + output, + console, + n_obs=n_obs, + n_var=n_var, + obs_names=obs_names, + var_names=var_names, + zarr_format=zarr_format, + force=force, + ) + except Exception as e: + console.print(f"[bold red]Error:[/] {e}") + raise typer.Exit(code=1) + + # ============================================================================ # LS command # ============================================================================ @@ -209,6 +282,17 @@ def subset( exists=True, readable=True, ), + obs_query: Optional[str] = typer.Option( + None, + "--obs-query", + "-q", + help="Keep obs matching an expression, e.g. \"cluster == Cortex_2\"", + ), + var_query: Optional[str] = typer.Option( + None, + "--var-query", + help="Keep var matching an expression, e.g. \"highly_variable == True\"", + ), chunk_rows: int = typer.Option( 1024, "--chunk", @@ -217,11 +301,29 @@ def subset( "-r", help="Row chunk size for dense matrices", ), + zarr_format: Optional[int] = typer.Option( + None, + "--zarr-format", + help="Zarr spec version to write (defaults to the source store's)", + ), ) -> None: - """Subset an AnnData store by obs and/or var names.""" - if obs is None and var is None: + """ + Subset an AnnData store by obs and/or var. + + Select either by name list (--obs/--var) or by expression + (--obs-query/--var-query). Expressions support ==, !=, <, <=, >, >=, in, + not in, and, or, not, and parentheses. + + Examples: + adata subset data.h5ad -o out.h5ad --obs barcodes.txt + adata subset data.h5ad -o out.h5ad --obs-query "cluster == Cortex_2" + adata subset data.h5ad -o out.h5ad -q "n_counts > 1000 and cluster in A,B" + adata subset data.h5ad -o out.h5ad --var-query "highly_variable == True" + """ + if obs is None and var is None and obs_query is None and var_query is None: console.print( - "[bold red]Error:[/] At least one of --obs or --var must be provided.", + "[bold red]Error:[/] Provide at least one of --obs, --var, " + "--obs-query or --var-query.", ) raise typer.Exit(code=1) @@ -241,6 +343,175 @@ def subset( chunk_rows=chunk_rows, console=console, inplace=inplace, + obs_query=obs_query, + var_query=var_query, + zarr_format=zarr_format, + ) + except Exception as e: + console.print(f"[bold red]Error:[/] {e}") + raise typer.Exit(code=1) + + +# ============================================================================ +# CONCAT command +# ============================================================================ +@app.command("concat") +def concat( + files: List[Path] = typer.Argument( + ..., + help="Two or more .h5ad/.zarr stores to concatenate", + exists=True, + readable=True, + dir_okay=True, + file_okay=True, + ), + output: Path = typer.Option( + ..., "--output", "-o", help="Output .h5ad/.zarr path", dir_okay=True + ), + join: str = typer.Option( + "inner", + "--join", + "-j", + help="Align var by intersection ('inner') or union ('outer')", + ), + label: Optional[str] = typer.Option( + None, "--label", help="Add an obs column recording which input each cell came from" + ), + keys: Optional[str] = typer.Option( + None, + "--keys", + help="Comma separated names for the inputs (defaults to their filenames)", + ), + index_unique: Optional[str] = typer.Option( + None, + "--index-unique", + help="Delimiter used to suffix obs names with their key, e.g. '-'", + ), + merge: Optional[str] = typer.Option( + None, + "--merge", + help="How to reconcile var columns: same, unique, first, only (default: drop)", + ), + uns_merge: Optional[str] = typer.Option( + None, + "--uns-merge", + help="How to reconcile uns: same, unique, first, only (default: drop)", + ), + fill_value: float = typer.Option( + 0.0, "--fill-value", help="Value for dense cells introduced by an outer join" + ), + chunk_rows: int = typer.Option( + 1024, "--chunk", "-C", help="Row chunk size while streaming" + ), + zarr_format: Optional[int] = typer.Option( + None, "--zarr-format", help="Zarr spec version to write (2 or 3)" + ), +) -> None: + """ + Concatenate stores along the obs axis. + + Streams each input in turn, so memory use is set by --chunk rather than by + the size of the inputs. obsp, varp and raw are not carried over. + + Examples: + adata concat a.h5ad b.h5ad -o merged.h5ad + adata concat *.h5ad -o merged.h5ad --join outer --label sample + adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge same + """ + for name, value in (("--merge", merge), ("--uns-merge", uns_merge)): + if value is not None and value not in MERGE_STRATEGIES: + console.print( + f"[bold red]Error:[/] {name} must be one of: " + f"{', '.join(MERGE_STRATEGIES)}" + ) + raise typer.Exit(code=1) + + try: + concat_stores( + files, + output, + console, + join=join, + label=label, + keys=[k.strip() for k in keys.split(",")] if keys else None, + index_unique=index_unique, + merge=merge, + uns_merge=uns_merge, + fill_value=fill_value, + chunk_rows=chunk_rows, + zarr_format=zarr_format, + ) + except Exception as e: + console.print(f"[bold red]Error:[/] {e}") + raise typer.Exit(code=1) + + +# ============================================================================ +# SPLIT command +# ============================================================================ +@app.command("split") +def split( + file: Path = typer.Argument( + ..., + help="Input .h5ad/.zarr", + exists=True, + readable=True, + dir_okay=True, + file_okay=True, + ), + by: str = typer.Option( + ..., "--by", "-b", help="Column to split on (e.g. 'sample', 'cell_type')" + ), + output_dir: Path = typer.Option( + ..., "--output-dir", "-o", help="Directory to write the split stores into" + ), + axis: str = typer.Option( + "obs", "--axis", help="Axis the column belongs to ('obs' or 'var')" + ), + suffix: Optional[str] = typer.Option( + None, + "--suffix", + help="Output extension (defaults to the source store's format)", + ), + min_size: int = typer.Option( + 1, "--min-size", help="Skip groups with fewer rows than this" + ), + manifest: bool = typer.Option( + True, "--manifest/--no-manifest", help="Write a CSV manifest of the outputs" + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Show what would be written without writing it" + ), + chunk_rows: int = typer.Option( + 1024, "--chunk", "-C", help="Row chunk size for dense matrices" + ), +) -> None: + """ + Split a store into one file per distinct value of a column. + + Streams the source, so it works on stores far larger than memory. + + Examples: + adata split data.h5ad --by sample -o per_sample/ + adata split data.h5ad --by cell_type -o clusters/ --dry-run + adata split data.h5ad --by cluster -o out/ --min-size 50 + """ + if axis not in ("obs", "var"): + console.print("[bold red]Error:[/] --axis must be 'obs' or 'var'.") + raise typer.Exit(code=1) + + try: + split_store( + file=file, + column=by, + output_dir=output_dir, + console=console, + axis=axis, + suffix=suffix, + dry_run=dry_run, + manifest=manifest, + min_size=min_size, + chunk_rows=chunk_rows, ) except Exception as e: console.print(f"[bold red]Error:[/] {e}") @@ -528,7 +799,7 @@ def import_dataframe( file_okay=True, ), entry: str = typer.Argument( - ..., help="Entry path to create/replace ('obs' or 'var')" + ..., help="Dataframe path to create/replace (e.g. 'obs', 'var', 'raw/var')" ), input_file: Path = typer.Argument( ..., help="Input CSV file", exists=True, readable=True @@ -552,22 +823,31 @@ def import_dataframe( "-i", help="Column to use as index. Defaults to first column.", ), + categorical: Optional[str] = typer.Option( + None, + "--categorical", + help="Comma separated columns to force to categorical", + ), + auto_categorical: bool = typer.Option( + True, + "--auto-categorical/--no-auto-categorical", + help="Infer categoricals from low-cardinality string columns", + ), ) -> None: """ - Import a CSV file into obs or var. + Import a CSV file as a dataframe. + + String columns with few distinct values become categoricals; pass + --no-auto-categorical to keep them as plain strings, or --categorical to + force specific ones. Examples: - h5ad import dataframe data.h5ad obs cells.csv -o output.h5ad -i cell_id - h5ad import dataframe data.h5ad var genes.csv --inplace -i gene_id + adata import dataframe data.h5ad obs cells.csv -o out.h5ad -i cell_id + adata import dataframe data.h5ad var genes.csv --inplace -i gene_id + adata import dataframe data.h5ad obs cells.csv --inplace --categorical batch """ from adata.commands.import_data import _import_csv - if entry not in ("obs", "var"): - console.print( - f"[bold red]Error:[/] Entry must be 'obs' or 'var', not '{entry}'.", - ) - raise typer.Exit(code=1) - if not inplace and output is None: console.print( "[bold red]Error:[/] Output file is required. " @@ -577,7 +857,20 @@ def import_dataframe( try: target = _get_target_file(file, output, inplace) - _import_csv(target, entry, input_file, index_column, console) + cat_list = ( + [c.strip() for c in categorical.split(",") if c.strip()] + if categorical + else None + ) + _import_csv( + target, + entry, + input_file, + index_column, + console, + categorical=cat_list, + auto_categorical=auto_categorical, + ) except Exception as e: console.print(f"[bold red]Error:[/] {e}") raise typer.Exit(code=1) @@ -749,6 +1042,57 @@ def import_dict( raise typer.Exit(code=1) +@import_app.command("image") +def import_image_cmd( + file: Path = typer.Argument( + ..., + help="Path to the source .h5ad/.zarr store", + exists=True, + readable=True, + dir_okay=True, + file_okay=True, + ), + entry: str = typer.Argument( + ..., help="Entry path to create/replace (e.g. 'uns/spatial/hires')" + ), + input_file: Path = typer.Argument( + ..., help="Input image file (.png, .jpg, .tiff)", exists=True, readable=True + ), + output: Optional[Path] = typer.Option( + None, + "--output", + "-o", + help="Output .h5ad/.zarr path. Required unless --inplace.", + dir_okay=True, + file_okay=True, + ), + inplace: bool = typer.Option( + False, "--inplace", help="Modify source file directly." + ), +) -> None: + """ + Import an image file as a dense array. + + Examples: + adata import image data.h5ad uns/spatial/hires tissue.png --inplace + """ + from adata.commands.import_data import _import_image + + if not inplace and output is None: + console.print( + "[bold red]Error:[/] Output file is required. " + "Use --output/-o or --inplace.", + ) + raise typer.Exit(code=1) + + try: + target = _get_target_file(file, output, inplace) + _import_image(target, entry, input_file, console) + except Exception as e: + console.print(f"[bold red]Error:[/] {e}") + raise typer.Exit(code=1) + + def main(argv: Optional[Sequence[str]] = None) -> None: app(standalone_mode=True) diff --git a/src/adata/commands/__init__.py b/src/adata/commands/__init__.py index 2f15de5..70e0690 100644 --- a/src/adata/commands/__init__.py +++ b/src/adata/commands/__init__.py @@ -3,3 +3,6 @@ from adata.commands.export import export_table, export_image, export_json, export_mtx, export_npy from adata.commands.import_data import import_object from adata.commands.ls import list_store +from adata.commands.create import create_store +from adata.commands.split import split_store +from adata.commands.concat import MERGE_STRATEGIES, concat_stores diff --git a/src/adata/commands/concat.py b/src/adata/commands/concat.py new file mode 100644 index 0000000..650b854 --- /dev/null +++ b/src/adata/commands/concat.py @@ -0,0 +1,21 @@ +"""Command wrapper for `concat`.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Optional, Sequence + +from rich.console import Console + +from adata.core.concat import MERGE_STRATEGIES, concat_on_disk + +__all__ = ["MERGE_STRATEGIES", "concat_stores"] + + +def concat_stores( + files: Sequence[Path], + output: Path, + console: Console, + **kwargs: object, +) -> None: + concat_on_disk(files, output, console, **kwargs) # type: ignore[arg-type] diff --git a/src/adata/commands/create.py b/src/adata/commands/create.py new file mode 100644 index 0000000..b82768f --- /dev/null +++ b/src/adata/commands/create.py @@ -0,0 +1,93 @@ +"""`create` -- write a new, empty AnnData store. + +Gives `import` something to write into. The result is a complete AnnData +object in its own right: root attributes, obs and var dataframes with real +indices, and the optional mapping groups, so anndata can open it before a +single matrix has been attached. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import List, Optional + +from rich.console import Console + +from adata.elements.write import ensure_anndata_skeleton, write_dataframe_header +from adata.storage import open_store + + +def _read_names(path: Path) -> List[str]: + """Read a newline-delimited name list, ignoring blank lines.""" + names = [ + line.strip() + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if not names: + raise ValueError(f"'{path}' contains no names.") + if len(set(names)) != len(names): + raise ValueError(f"'{path}' contains duplicate names.") + return names + + +def _resolve_axis( + axis: str, n: Optional[int], names_file: Optional[Path] +) -> List[str]: + """Determine an axis's index, from an explicit list or a generated range.""" + if names_file is not None: + names = _read_names(names_file) + if n is not None and n != len(names): + raise ValueError( + f"--n-{axis} is {n} but '{names_file}' has {len(names)} names." + ) + return names + + if n is None: + raise ValueError(f"Provide either --n-{axis} or --{axis}-names.") + if n < 0: + raise ValueError(f"--n-{axis} must not be negative.") + + width = len(str(max(n - 1, 0))) + prefix = "cell" if axis == "obs" else "gene" + return [f"{prefix}_{i:0{width}d}" for i in range(n)] + + +def create_store( + output: Path, + console: Console, + n_obs: Optional[int] = None, + n_var: Optional[int] = None, + obs_names: Optional[Path] = None, + var_names: Optional[Path] = None, + zarr_format: Optional[int] = None, + force: bool = False, +) -> None: + """Create an empty AnnData store at `output`. + + Names come from `obs_names`/`var_names` when given, otherwise they are + generated as `cell_0000`-style labels wide enough for the axis length. + """ + if output.exists() and not force: + raise FileExistsError( + f"'{output}' already exists. Pass --force to overwrite it." + ) + + obs_index = _resolve_axis("obs", n_obs, obs_names) + var_index = _resolve_axis("var", n_var, var_names) + + if output.exists() and force: + import shutil + + shutil.rmtree(output) if output.is_dir() else output.unlink() + + with open_store(output, "w", zarr_format=zarr_format) as store: + root = store.root + write_dataframe_header(root, "obs", obs_index, []) + write_dataframe_header(root, "var", var_index, []) + ensure_anndata_skeleton(root) + + console.print( + f"[green]Created[/] {output} " + f"({len(obs_index)} obs x {len(var_index)} var, empty)" + ) diff --git a/src/adata/commands/import_data.py b/src/adata/commands/import_data.py index b47c330..9fae7cb 100644 --- a/src/adata/commands/import_data.py +++ b/src/adata/commands/import_data.py @@ -9,6 +9,7 @@ from adata.formats.array import import_npy from adata.formats.dataframe import import_dataframe +from adata.formats.image import import_image from adata.formats.json_data import import_json from adata.formats.sparse import import_mtx from adata.storage import copy_path, copy_store_contents, detect_backend, open_store @@ -88,6 +89,8 @@ def _import_csv( input_file: Path, index_column: Optional[str], console: Console, + categorical: Optional[list] = None, + auto_categorical: bool = True, ) -> None: with open_store(file, "a") as store: import_dataframe( @@ -96,6 +99,8 @@ def _import_csv( input_file=input_file, index_column=index_column, console=console, + categorical=categorical, + auto_categorical=auto_categorical, ) @@ -119,6 +124,16 @@ def _import_mtx( import_mtx(store.root, obj=obj, input_file=input_file, console=console) +def _import_image( + file: Path, + obj: str, + input_file: Path, + console: Console, +) -> None: + with open_store(file, "a") as store: + import_image(store.root, obj=obj, input_file=input_file, console=console) + + def _import_json( file: Path, obj: str, diff --git a/src/adata/commands/split.py b/src/adata/commands/split.py new file mode 100644 index 0000000..db9ddc3 --- /dev/null +++ b/src/adata/commands/split.py @@ -0,0 +1,147 @@ +"""`split` -- write one store per distinct value of an annotation column. + +Requested in issue #2, modelled on cellgeni/scraft's `split_h5ad` but +streaming: the source is never loaded into memory, only the column being split +on is read, and each output is produced by the same subset machinery used by +`adata subset`. +""" + +from __future__ import annotations + +import csv +import re +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +from rich.console import Console + +from adata.core.select import group_indices +from adata.core.subset import subset_h5ad +from adata.storage import detect_backend, open_store + + +def sanitize_label(label: str) -> str: + """Turn a column value into something safe to use as a filename.""" + text = (label or "").strip() + if not text: + return "NA" + text = text.replace("/", "_").replace("\\", "_") + text = re.sub(r"\s+", "_", text) + text = re.sub(r"[^A-Za-z0-9_.-]+", "", text) + return text or "NA" + + +def unique_names(labels: List[str]) -> Dict[str, str]: + """Map each label to a distinct filename stem. + + Sanitising can map two different labels onto the same stem ("a/b" and + "a b"), so collisions get a numeric suffix rather than silently + overwriting one another. + """ + used: Dict[str, int] = {} + mapping: Dict[str, str] = {} + for label in labels: + base = sanitize_label(label) + count = used.get(base, 0) + used[base] = count + 1 + mapping[label] = base if count == 0 else f"{base}_{count}" + return mapping + + +def split_store( + file: Path, + column: str, + output_dir: Path, + console: Console, + axis: str = "obs", + suffix: Optional[str] = None, + dry_run: bool = False, + manifest: bool = True, + min_size: int = 1, + chunk_rows: int = 1024, +) -> List[Tuple[str, Path, int]]: + """Split `file` into one store per distinct value of `column`. + + Returns ``(label, path, n_rows)`` for each group written. Groups smaller + than `min_size` are skipped with a warning rather than producing tiny + stores nobody asked for. + """ + if axis not in ("obs", "var"): + raise ValueError("--axis must be 'obs' or 'var'.") + + if suffix is None: + suffix = ".zarr" if detect_backend(file) == "zarr" else ".h5ad" + + with open_store(file, "r") as store: + groups, order = group_indices(store.root, axis, column) + zarr_format = store.zarr_format + + if not groups: + raise ValueError(f"Column {column!r} produced no groups.") + + names = unique_names(order) + console.print( + f"[cyan]Splitting on {axis}[{column}]: " + f"{len(order)} group{'s' if len(order) != 1 else ''}[/]" + ) + + planned: List[Tuple[str, Path, int]] = [] + for label in order: + indices = groups[label] + out_path = output_dir / f"{names[label]}{suffix}" + if len(indices) < min_size: + console.print( + f"[yellow]Skipping {label!r}: {len(indices)} " + f"{axis} < --min-size {min_size}[/]" + ) + continue + planned.append((label, out_path, len(indices))) + + for label, out_path, count in planned: + console.print(f" {label!r} -> {out_path} ({count} {axis})") + + if dry_run: + console.print("[yellow]Dry run: nothing written.[/]") + return planned + + output_dir.mkdir(parents=True, exist_ok=True) + + for label, out_path, _ in planned: + indices = np.sort(groups[label]) + subset_h5ad( + file=file, + output=out_path, + obs_file=None, + var_file=None, + chunk_rows=chunk_rows, + console=console, + obs_indices=indices if axis == "obs" else None, + var_indices=indices if axis == "var" else None, + zarr_format=zarr_format, + ) + + if manifest: + _write_manifest(file, output_dir, column, planned, console) + + console.print(f"[green]Wrote[/] {len(planned)} stores to {output_dir}") + return planned + + +def _write_manifest( + file: Path, + output_dir: Path, + column: str, + planned: List[Tuple[str, Path, int]], + console: Console, +) -> Path: + """Record what was written, for downstream pipelines to consume.""" + source_id = file.stem if file.suffix else file.name + path = output_dir / f"{source_id}_manifest.csv" + with open(path, "w", newline="", encoding="utf-8") as fh: + writer = csv.writer(fh) + writer.writerow(["id", "column", "value", "anndatas", "n"]) + for label, out_path, count in planned: + writer.writerow([source_id, column, label, str(out_path), count]) + console.print(f"[dim]Manifest written to {path}[/]") + return path diff --git a/src/adata/core/concat.py b/src/adata/core/concat.py new file mode 100644 index 0000000..4b6cea9 --- /dev/null +++ b/src/adata/core/concat.py @@ -0,0 +1,858 @@ +"""Concatenating AnnData stores on disk, along the obs axis. + +Mirrors the useful core of `anndata.experimental.concat_on_disk`: inputs are +streamed a block of rows at a time and written straight into the output, so +peak memory is set by the block size rather than by the total size of the +inputs. + +Columns of the concatenated axis follow `join`; elements of the alternative +axis and of uns follow a `merge` strategy, since there is no single correct +way to reconcile values that differ between inputs. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +import numpy as np +from rich.console import Console + +from adata.core.info import axis_len +from adata.core.select import read_names +from adata.elements import spec +from adata.elements.read import ( + dataframe_columns, + element_len, + read_str_all, + read_categories, + is_ordered, + resolve_index, +) +from adata.elements.write import ( + ensure_anndata_skeleton, + set_column_order, + write_categorical, + write_dataframe_header, + write_dense, + write_masked, + write_mapping, + write_string_array, +) +from adata.storage import ( + copy_tree, + create_dataset, + is_dataset, + is_group, + open_store, +) + +MERGE_STRATEGIES = ("same", "unique", "first", "only") + + +def _index_union(per_input: Sequence[List[str]]) -> List[str]: + """Union of indices, in order of first appearance.""" + seen: Dict[str, None] = {} + for names in per_input: + for name in names: + seen.setdefault(name, None) + return list(seen) + + +def _index_intersection(per_input: Sequence[List[str]]) -> List[str]: + """Intersection of indices, ordered by the first input.""" + common = set(per_input[0]) + for names in per_input[1:]: + common &= set(names) + return [name for name in per_input[0] if name in common] + + +def _column_map(source: List[str], target: List[str]) -> np.ndarray: + """Map each source position to its target position, or -1 if dropped.""" + lookup = {name: i for i, name in enumerate(target)} + return np.fromiter( + (lookup.get(name, -1) for name in source), dtype=np.int64, count=len(source) + ) + + +def _apply_index_unique( + names: List[str], key: Optional[str], delimiter: Optional[str] +) -> List[str]: + if delimiter is None or key is None: + return names + return [f"{name}{delimiter}{key}" for name in names] + + +def _resolve_keys( + files: Sequence[Path], keys: Optional[Sequence[str]] +) -> List[str]: + if keys is None: + return [f.stem if f.suffix else f.name for f in files] + if len(keys) != len(files): + raise ValueError( + f"--keys has {len(keys)} entries but {len(files)} inputs were given." + ) + return list(keys) + + +# --------------------------------------------------------------------------- +# merge strategies + + +def _merge_values(values: List[Any], strategy: Optional[str]) -> Tuple[bool, Any]: + """Reconcile one element's value across inputs. + + Returns ``(keep, value)``. The strategies match anndata's: "same" keeps a + value only when every input agrees, "unique" when there is exactly one + distinct value among those present, "first" takes the earliest present, + and "only" keeps it when exactly one input has it at all. + """ + if strategy is None: + return False, None + + present = [v for v in values if v is not _MISSING] + if not present: + return False, None + + if strategy == "first": + return True, present[0] + + if strategy == "only": + return (len(present) == 1), (present[0] if len(present) == 1 else None) + + distinct: List[Any] = [] + for value in present: + if not any(_equal(value, seen) for seen in distinct): + distinct.append(value) + + if strategy == "unique": + return (len(distinct) == 1), (distinct[0] if len(distinct) == 1 else None) + + if strategy == "same": + keep = len(distinct) == 1 and len(present) == len(values) + return keep, (distinct[0] if keep else None) + + raise ValueError( + f"Unknown merge strategy {strategy!r}. " + f"Choose from: {', '.join(MERGE_STRATEGIES)}" + ) + + +class _Missing: + def __repr__(self) -> str: # pragma: no cover - debugging aid + return "" + + +_MISSING = _Missing() + + +def _equal(a: Any, b: Any) -> bool: + try: + if isinstance(a, np.ndarray) or isinstance(b, np.ndarray): + return np.array_equal(np.asarray(a), np.asarray(b)) + return bool(a == b) + except Exception: + return False + + +# --------------------------------------------------------------------------- +# obs columns + + +def _column_kind(obj: Any) -> str: + """Classify a column by how it must be concatenated.""" + enc = spec.encoding_type(obj) + if enc == spec.CATEGORICAL or (is_group(obj) and "codes" in obj): + return "categorical" + if enc in spec.MASKED_TYPES or (is_group(obj) and "values" in obj): + return "masked" + if is_dataset(obj): + from adata.elements.strings import is_string_dtype + + return "string" if is_string_dtype(obj.dtype) else "numeric" + return "other" + + +def _concat_categorical( + parent: Any, name: str, columns: List[Optional[Any]], lengths: List[int] +) -> None: + """Concatenate categorical columns, unioning their category sets. + + Inputs rarely share a category order, so codes are remapped rather than + concatenated directly. A row from an input lacking the column gets code + -1, which anndata reads back as a missing value. + """ + categories: List[str] = [] + for col in columns: + if col is None: + continue + for category in read_categories(col): + if category not in categories: + categories.append(str(category)) + + lookup = {c: i for i, c in enumerate(categories)} + ordered = all(is_ordered(c) for c in columns if c is not None) + + codes = np.full(sum(lengths), -1, dtype=np.int64) + offset = 0 + for col, length in zip(columns, lengths): + if col is not None: + source_cats = [str(c) for c in read_categories(col)] + remap = np.array( + [lookup[c] for c in source_cats] + [-1], dtype=np.int64 + ) + source_codes = np.asarray(col["codes"][...], dtype=np.int64) + source_codes[source_codes < 0] = len(source_cats) + codes[offset : offset + length] = remap[source_codes] + offset += length + + write_categorical(parent, name, codes, categories, ordered=ordered) + + +def _concat_numeric( + parent: Any, name: str, columns: List[Optional[Any]], lengths: List[int] +) -> None: + """Concatenate numeric columns, promoting to float when rows are missing. + + An integer column cannot represent "absent", so a column missing from some + input is promoted to float and padded with NaN -- the same thing pandas + does on an outer join. + """ + present = [c for c in columns if c is not None] + dtype = np.result_type(*[c.dtype for c in present]) + complete = len(present) == len(columns) + + if not complete and dtype.kind in ("i", "u", "b"): + dtype = np.dtype("float64") + + out = np.empty(sum(lengths), dtype=dtype) + if not complete: + out[:] = np.nan if dtype.kind == "f" else 0 + + offset = 0 + for col, length in zip(columns, lengths): + if col is not None: + out[offset : offset + length] = np.asarray(col[...], dtype=dtype) + offset += length + + write_dense(parent, name, out) + + +def _concat_masked( + parent: Any, name: str, columns: List[Optional[Any]], lengths: List[int] +) -> None: + """Concatenate nullable columns, masking rows from inputs that lack them.""" + kinds = {spec.encoding_type(c) for c in columns if c is not None} + enc = kinds.pop() if len(kinds) == 1 else spec.NULLABLE_STRING_ARRAY + + total = sum(lengths) + mask = np.ones(total, dtype=bool) + values: List[Any] = [None] * total + + offset = 0 + for col, length in zip(columns, lengths): + if col is not None: + chunk = np.asarray(col["values"][...]) + chunk_mask = np.asarray(col["mask"][...], dtype=bool) + for i in range(length): + values[offset + i] = chunk[i] + mask[offset : offset + length] = chunk_mask + offset += length + + if enc == spec.NULLABLE_STRING_ARRAY: + from adata.elements.read import decode_str_array + + filled = [ + "" if v is None else decode_str_array(np.asarray([v]))[0] for v in values + ] + write_masked(parent, name, filled, mask, enc) + return + + present = [c for c in columns if c is not None] + dtype = np.result_type(*[c["values"].dtype for c in present]) + filled_num = np.array( + [0 if v is None else v for v in values], dtype=dtype + ) + write_masked(parent, name, filled_num, mask, enc) + + +def _concat_string( + parent: Any, name: str, columns: List[Optional[Any]], lengths: List[int] +) -> None: + """Concatenate string columns, masking rows from inputs that lack them.""" + total = sum(lengths) + values: List[str] = [""] * total + mask = np.zeros(total, dtype=bool) + + offset = 0 + for col, length in zip(columns, lengths): + if col is None: + mask[offset : offset + length] = True + else: + values[offset : offset + length] = read_str_all(col) + offset += length + + if mask.any(): + write_masked(parent, name, values, mask, spec.NULLABLE_STRING_ARRAY) + else: + write_string_array(parent, name, values) + + +def _concat_obs_column( + parent: Any, + name: str, + columns: List[Optional[Any]], + lengths: List[int], + console: Console, +) -> bool: + """Write one concatenated annotation column. Returns whether it was kept.""" + kinds = {_column_kind(c) for c in columns if c is not None} + + if kinds == {"categorical"}: + _concat_categorical(parent, name, columns, lengths) + elif kinds == {"numeric"}: + _concat_numeric(parent, name, columns, lengths) + elif kinds == {"masked"}: + _concat_masked(parent, name, columns, lengths) + elif kinds <= {"string", "categorical", "masked"}: + # Mixed text-like encodings: fall back to plain strings, which every + # one of them can represent. + _concat_string(parent, name, columns, lengths) + else: + console.print( + f"[yellow]Dropping column {name!r}: " + f"cannot concatenate encodings {sorted(kinds)}[/]" + ) + return False + return True + + +# --------------------------------------------------------------------------- +# matrices + + +def _matrix_kind(obj: Any) -> str: + enc = spec.encoding_type(obj) + if enc in spec.SPARSE_TYPES: + return enc + if is_dataset(obj): + return "dense" + return "other" + + +def _concat_sparse( + dst_parent: Any, + name: str, + sources: List[Any], + col_maps: List[np.ndarray], + n_rows: int, + n_cols: int, + chunk_rows: int, +) -> None: + """Concatenate CSR matrices row-wise, remapping columns as they stream. + + Absent columns need no fill: a sparse matrix's zeros are implicit, so + dropped entries simply do not appear in the output. + """ + from adata.core.subset import _append, _growable + + group = dst_parent.create_group(name) + spec.set_encoding(group, spec.CSR_MATRIX) + from adata.elements.write import set_shape_attr + + set_shape_attr(group, (n_rows, n_cols)) + + dtype = np.result_type(*[s["data"].dtype for s in sources]) + out_data = _growable(group, "data", dtype) + out_indices = _growable(group, "indices", np.int64) + indptr = [0] + nnz = 0 + + for source, col_map in zip(sources, col_maps): + src_indptr = np.asarray(source["indptr"][...], dtype=np.int64) + src_data, src_indices = source["data"], source["indices"] + n_src_rows = len(src_indptr) - 1 + + for start in range(0, n_src_rows, chunk_rows): + end = min(start + chunk_rows, n_src_rows) + lo, hi = int(src_indptr[start]), int(src_indptr[end]) + block_idx = ( + np.asarray(src_indices[lo:hi], dtype=np.int64) + if hi > lo + else np.empty(0, dtype=np.int64) + ) + block_data = ( + np.asarray(src_data[lo:hi]) if hi > lo else np.empty(0, dtype=dtype) + ) + + kept_idx: List[np.ndarray] = [] + kept_data: List[np.ndarray] = [] + for row in range(start, end): + sl = slice(int(src_indptr[row]) - lo, int(src_indptr[row + 1]) - lo) + mapped = col_map[block_idx[sl]] + keep = mapped >= 0 + kept_idx.append(mapped[keep]) + kept_data.append(block_data[sl][keep]) + nnz += int(keep.sum()) + indptr.append(nnz) + + if kept_idx: + _append(out_indices, np.concatenate(kept_idx)) + _append(out_data, np.concatenate(kept_data).astype(dtype)) + + create_dataset(group, "indptr", data=np.asarray(indptr, dtype=np.int64)) + + +def _concat_dense( + dst_parent: Any, + name: str, + sources: List[Any], + col_maps: List[np.ndarray], + n_rows: int, + n_cols: int, + chunk_rows: int, + fill_value: float, +) -> None: + """Concatenate dense matrices row-wise, scattering columns into place.""" + dtype = np.result_type(*[s.dtype for s in sources]) + if fill_value != 0 and dtype.kind in ("i", "u", "b"): + dtype = np.dtype("float64") + + out = create_dataset(dst_parent, name, shape=(n_rows, n_cols), dtype=dtype) + spec.set_encoding(out, spec.ARRAY) + + row_offset = 0 + for source, col_map in zip(sources, col_maps): + keep = col_map >= 0 + targets = col_map[keep] + n_src_rows = source.shape[0] + + for start in range(0, n_src_rows, chunk_rows): + end = min(start + chunk_rows, n_src_rows) + block = np.full((end - start, n_cols), fill_value, dtype=dtype) + block[:, targets] = np.asarray(source[start:end, :])[:, keep] + out[row_offset + start : row_offset + end, :] = block + + row_offset += n_src_rows + + +def _concat_matrix( + dst_parent: Any, + name: str, + sources: List[Any], + col_maps: List[np.ndarray], + n_rows: int, + n_cols: int, + chunk_rows: int, + fill_value: float, + console: Console, +) -> bool: + """Concatenate X or one layer across inputs. Returns whether it was written.""" + kinds = {_matrix_kind(s) for s in sources} + + if kinds == {spec.CSR_MATRIX}: + _concat_sparse( + dst_parent, name, sources, col_maps, n_rows, n_cols, chunk_rows + ) + return True + + if kinds == {"dense"}: + _concat_dense( + dst_parent, + name, + sources, + col_maps, + n_rows, + n_cols, + chunk_rows, + fill_value, + ) + return True + + console.print( + f"[yellow]Skipping {name!r}: inputs disagree on encoding " + f"({', '.join(sorted(kinds))}). Convert them to match first.[/]" + ) + return False + + +def _concat_obsm( + dst_parent: Any, + name: str, + sources: List[Any], + n_rows: int, + chunk_rows: int, + console: Console, +) -> bool: + """Concatenate an obsm entry row-wise; its columns are not an axis.""" + if not all(is_dataset(s) for s in sources): + console.print(f"[yellow]Skipping obsm/{name}: not a dense array in every input[/]") + return False + + widths = {tuple(s.shape[1:]) for s in sources} + if len(widths) != 1: + console.print( + f"[yellow]Skipping obsm/{name}: inputs disagree on shape {sorted(widths)}[/]" + ) + return False + + trailing = widths.pop() + dtype = np.result_type(*[s.dtype for s in sources]) + out = create_dataset( + dst_parent, name, shape=(n_rows,) + trailing, dtype=dtype + ) + spec.set_encoding(out, spec.ARRAY) + + offset = 0 + for source in sources: + for start in range(0, source.shape[0], chunk_rows): + end = min(start + chunk_rows, source.shape[0]) + out[offset + start : offset + end, ...] = source[start:end, ...] + offset += source.shape[0] + return True + + +# --------------------------------------------------------------------------- +# orchestration + + +def _merge_group( + dst_parent: Any, + name: str, + groups: List[Optional[Any]], + strategy: Optional[str], + console: Console, +) -> None: + """Merge a mapping (uns, varm, ...) across inputs under `strategy`.""" + target = write_mapping(dst_parent, name) + if strategy is None: + return + + keys: List[str] = [] + for group in groups: + if group is None: + continue + for key in group.keys(): + if key not in keys: + keys.append(key) + + for key in keys: + members = [ + g[key] if g is not None and key in g else _MISSING for g in groups + ] + present = [m for m in members if m is not _MISSING] + + if all(is_group(m) for m in present) and strategy in ("same", "unique"): + # Nested mappings are merged member-wise rather than compared whole. + if all(spec.encoding_type(m) == spec.DICT for m in present): + _merge_group(target, key, [ + m if m is not _MISSING else None for m in members + ], strategy, console) + continue + + comparable = [ + _MISSING if m is _MISSING else _readable_value(m) for m in members + ] + keep, _ = _merge_values(comparable, strategy) + if keep: + source = next(m for m in members if m is not _MISSING) + copy_tree(source, target, key) + + +def _readable_value(obj: Any) -> Any: + """A comparable snapshot of a small element, for merge strategies.""" + try: + if is_dataset(obj): + return np.asarray(obj[...]) + return tuple(sorted(obj.keys())) + except Exception: + return _MISSING + + +def concat_on_disk( + files: Sequence[Path], + output: Path, + console: Console, + *, + join: str = "inner", + label: Optional[str] = None, + keys: Optional[Sequence[str]] = None, + index_unique: Optional[str] = None, + merge: Optional[str] = None, + uns_merge: Optional[str] = None, + fill_value: float = 0.0, + chunk_rows: int = 1024, + zarr_format: Optional[int] = None, +) -> None: + """Concatenate stores along the obs axis, streaming each one in turn. + + `join` aligns var: "inner" keeps only variables present in every input, + "outer" keeps their union. obs columns follow the same rule. Elements of + the var axis and of uns are reconciled by `merge`/`uns_merge`, since + inputs may legitimately disagree about them. + + obsp and varp are not concatenated -- a pairwise matrix has no meaningful + value between cells that came from different inputs. anndata's own + implementation refuses these too. + """ + if len(files) < 2: + raise ValueError("Concatenation needs at least two input stores.") + if join not in ("inner", "outer"): + raise ValueError("--join must be 'inner' or 'outer'.") + if output.exists(): + raise FileExistsError(f"'{output}' already exists.") + + input_keys = _resolve_keys(files, keys) + stores = [open_store(f, "r") for f in files] + + try: + roots = [s.root for s in stores] + + var_names = [read_names(r, "var") for r in roots] + obs_names = [read_names(r, "obs") for r in roots] + obs_counts = [axis_len(r, "obs") for r in roots] + + target_var = ( + _index_intersection(var_names) + if join == "inner" + else _index_union(var_names) + ) + if not target_var: + raise ValueError( + "No variables are shared by all inputs. Use --join outer to " + "keep their union instead." + ) + + col_maps = [_column_map(names, target_var) for names in var_names] + n_obs = sum(obs_counts) + n_var = len(target_var) + + console.print( + f"[cyan]Concatenating {len(files)} stores: " + f"{n_obs} obs x {n_var} var ({join} join)[/]" + ) + for f, key, count in zip(files, input_keys, obs_counts): + console.print(f" [dim]{key}[/]: {count} obs from {f.name}") + + target_obs: List[str] = [] + for names, key in zip(obs_names, input_keys): + target_obs.extend(_apply_index_unique(names, key, index_unique)) + + if len(set(target_obs)) != len(target_obs): + console.print( + "[yellow]Warning: obs names are not unique across inputs. " + "Pass --index-unique to disambiguate them.[/]" + ) + + with open_store(output, "w", zarr_format=zarr_format) as dst_store: + dst = dst_store.root + _write_obs( + dst, roots, target_obs, obs_counts, join, label, input_keys, console + ) + _write_var(dst, roots, var_names, target_var, merge, console) + ensure_anndata_skeleton(dst) + + _write_matrices( + dst, roots, col_maps, n_obs, n_var, chunk_rows, fill_value, console + ) + _write_obsm(dst, roots, n_obs, chunk_rows, console) + + uns_groups = [r["uns"] if "uns" in r else None for r in roots] + if any(g is not None for g in uns_groups): + if uns_merge is None: + write_mapping(dst, "uns") + else: + _merge_group(dst, "uns", uns_groups, uns_merge, console) + + for name in ("obsp", "varp"): + if any(name in r and len(list(r[name].keys())) for r in roots): + console.print( + f"[yellow]Dropping {name}: pairwise values have no " + f"meaning across concatenated inputs.[/]" + ) + if any("raw" in r for r in roots): + console.print("[yellow]Dropping raw/: not concatenated.[/]") + + console.print(f"[green]Wrote[/] {output} ({n_obs} obs x {n_var} var)") + finally: + for store in stores: + store.close() + + +def _write_obs( + dst: Any, + roots: List[Any], + target_obs: List[str], + obs_counts: List[int], + join: str, + label: Optional[str], + input_keys: List[str], + console: Console, +) -> None: + """Write the concatenated obs frame, plus the batch column if requested.""" + groups = [r["obs"] for r in roots] + per_input_cols = [ + dataframe_columns(g, resolve_index(g, "obs")[1]) for g in groups + ] + + if join == "inner": + shared = set(per_input_cols[0]).intersection(*map(set, per_input_cols[1:])) + names = [c for c in per_input_cols[0] if c in shared] + else: + names = _index_union(per_input_cols) + + written: List[str] = [] + obs_group = write_dataframe_header(dst, "obs", target_obs, []) + + for name in names: + columns = [g[name] if name in g else None for g in groups] + if _concat_obs_column(obs_group, name, columns, obs_counts, console): + written.append(name) + + if label: + if label in written: + raise ValueError( + f"--label {label!r} collides with an existing obs column." + ) + codes = np.concatenate( + [np.full(n, i, dtype=np.int64) for i, n in enumerate(obs_counts)] + ) + write_categorical(obs_group, label, codes, input_keys, ordered=False) + written.append(label) + + set_column_order(obs_group, written) + + +def _write_var( + dst: Any, + roots: List[Any], + var_names: List[List[str]], + target_var: List[str], + merge: Optional[str], + console: Console, +) -> None: + """Write the aligned var frame, keeping columns the merge strategy allows.""" + var_group = write_dataframe_header(dst, "var", target_var, []) + if merge is None: + return + + groups = [r["var"] for r in roots] + per_input_cols = [ + dataframe_columns(g, resolve_index(g, "var")[1]) for g in groups + ] + candidates = _index_union(per_input_cols) + + positions = [_column_map(target_var, names) for names in var_names] + written: List[str] = [] + + for name in candidates: + aligned: List[Any] = [] + for group, where in zip(groups, positions): + if name not in group or (where < 0).any(): + aligned.append(_MISSING) + continue + aligned.append(tuple(read_str_all(group[name])[i] for i in where)) + + keep, _ = _merge_values(aligned, merge) + if not keep: + continue + + source_i = next( + i for i, v in enumerate(aligned) if v is not _MISSING + ) + column = groups[source_i][name] + take = positions[source_i] + _write_var_column(var_group, name, column, take) + written.append(name) + + set_column_order(var_group, written) + + +def _write_var_column( + parent: Any, name: str, column: Any, take: np.ndarray +) -> None: + """Write one var column, reordered onto the target var index.""" + kind = _column_kind(column) + if kind == "categorical": + categories = [str(c) for c in read_categories(column)] + codes = np.asarray(column["codes"][...], dtype=np.int64)[take] + write_categorical( + parent, name, codes, categories, ordered=is_ordered(column) + ) + elif kind == "numeric": + write_dense(parent, name, np.asarray(column[...])[take]) + else: + values = read_str_all(column) + write_string_array(parent, name, [values[i] for i in take]) + + +def _write_matrices( + dst: Any, + roots: List[Any], + col_maps: List[np.ndarray], + n_obs: int, + n_var: int, + chunk_rows: int, + fill_value: float, + console: Console, +) -> None: + """Concatenate X and every layer shared by all inputs.""" + if all("X" in r for r in roots): + _concat_matrix( + dst, + "X", + [r["X"] for r in roots], + col_maps, + n_obs, + n_var, + chunk_rows, + fill_value, + console, + ) + + layer_names = _index_union( + [list(r["layers"].keys()) if "layers" in r else [] for r in roots] + ) + if not layer_names: + return + + layers = write_mapping(dst, "layers") + for name in layer_names: + if not all("layers" in r and name in r["layers"] for r in roots): + console.print( + f"[yellow]Skipping layer {name!r}: not present in every input[/]" + ) + continue + _concat_matrix( + layers, + name, + [r["layers"][name] for r in roots], + col_maps, + n_obs, + n_var, + chunk_rows, + fill_value, + console, + ) + + +def _write_obsm( + dst: Any, roots: List[Any], n_obs: int, chunk_rows: int, console: Console +) -> None: + """Concatenate obsm entries shared by all inputs.""" + names = _index_union( + [list(r["obsm"].keys()) if "obsm" in r else [] for r in roots] + ) + if not names: + return + + obsm = write_mapping(dst, "obsm") + for name in names: + if not all("obsm" in r and name in r["obsm"] for r in roots): + console.print( + f"[yellow]Skipping obsm/{name}: not present in every input[/]" + ) + continue + _concat_obsm( + obsm, name, [r["obsm"][name] for r in roots], n_obs, chunk_rows, console + ) diff --git a/src/adata/core/query.py b/src/adata/core/query.py new file mode 100644 index 0000000..a54391c --- /dev/null +++ b/src/adata/core/query.py @@ -0,0 +1,268 @@ +"""A small predicate language for selecting obs/var rows. + +Deliberately not SQL. The point is to express the common filters -- one or two +comparisons on annotation columns -- without pulling a query engine into a tool +whose selling point is streaming with light dependencies. Anything more +involved is better served by exporting to CSV and using duckdb. + +Grammar:: + + expr := or_expr + or_expr := and_expr ( "or" and_expr )* + and_expr:= term ( "and" term )* + term := "not" term | "(" expr ")" | comparison + comparison := IDENT OP VALUE + OP := == | != | < | <= | > | >= | in | not in + +Values are bare words, quoted strings, or comma-separated lists for `in`. +Comparison against a column is done on its string form for equality and +membership, and numerically for the ordering operators. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Sequence + +import numpy as np + +_TOKEN = re.compile( + r""" + \s*(?: + (?P\() + | (?P\)) + | (?P==|!=|<=|>=|<|>) + | (?P"[^"]*"|'[^']*'|[^\s()]+) + ) + """, + re.VERBOSE, +) + +_KEYWORDS = {"and", "or", "not", "in"} + + +class QueryError(ValueError): + """Raised when a query cannot be parsed or refers to a missing column.""" + + +@dataclass +class Token: + kind: str + text: str + + +def tokenize(expr: str) -> List[Token]: + tokens: List[Token] = [] + pos = 0 + while pos < len(expr): + match = _TOKEN.match(expr, pos) + if match is None: + if expr[pos:].strip() == "": + break + raise QueryError(f"Cannot parse query at {expr[pos:]!r}") + pos = match.end() + for kind in ("lparen", "rparen", "op", "word"): + text = match.group(kind) + if text is None: + continue + if kind == "word" and text.lower() in _KEYWORDS: + tokens.append(Token(text.lower(), text.lower())) + else: + tokens.append(Token(kind, text)) + break + return tokens + + +def _unquote(text: str) -> str: + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + return text[1:-1] + return text + + +#: A predicate maps a column reader to a boolean mask over the chunk. +Predicate = Callable[[Dict[str, List[str]]], np.ndarray] + + +class _Parser: + def __init__(self, tokens: Sequence[Token]) -> None: + self.tokens = list(tokens) + self.pos = 0 + + def peek(self) -> Optional[Token]: + return self.tokens[self.pos] if self.pos < len(self.tokens) else None + + def take(self) -> Token: + token = self.peek() + if token is None: + raise QueryError("Unexpected end of query.") + self.pos += 1 + return token + + def parse(self) -> Predicate: + node = self.parse_or() + if self.peek() is not None: + raise QueryError(f"Unexpected {self.peek().text!r} in query.") + return node + + def parse_or(self) -> Predicate: + left = self.parse_and() + while self.peek() is not None and self.peek().kind == "or": + self.take() + right = self.parse_and() + left = _combine(left, right, np.logical_or) + return left + + def parse_and(self) -> Predicate: + left = self.parse_term() + while self.peek() is not None and self.peek().kind == "and": + self.take() + right = self.parse_term() + left = _combine(left, right, np.logical_and) + return left + + def parse_term(self) -> Predicate: + token = self.peek() + if token is None: + raise QueryError("Unexpected end of query.") + + if token.kind == "not": + self.take() + inner = self.parse_term() + return lambda cols: np.logical_not(inner(cols)) + + if token.kind == "lparen": + self.take() + inner = self.parse_or() + closing = self.take() + if closing.kind != "rparen": + raise QueryError("Unbalanced parentheses in query.") + return inner + + return self.parse_comparison() + + def parse_comparison(self) -> Predicate: + name_token = self.take() + if name_token.kind != "word": + raise QueryError(f"Expected a column name, got {name_token.text!r}.") + column = _unquote(name_token.text) + + op_token = self.take() + if op_token.kind == "not": + following = self.take() + if following.kind != "in": + raise QueryError("Expected 'in' after 'not'.") + return _membership(column, self._take_list(), negate=True) + if op_token.kind == "in": + return _membership(column, self._take_list(), negate=False) + if op_token.kind != "op": + raise QueryError( + f"Expected a comparison operator after {column!r}, " + f"got {op_token.text!r}." + ) + + value_token = self.take() + if value_token.kind != "word": + raise QueryError(f"Expected a value, got {value_token.text!r}.") + return _comparison(column, op_token.text, _unquote(value_token.text)) + + def _take_list(self) -> List[str]: + token = self.take() + if token.kind == "lparen": + items: List[str] = [] + while True: + nxt = self.take() + if nxt.kind == "rparen": + break + items.extend( + v.strip() for v in _unquote(nxt.text).split(",") if v.strip() + ) + return items + if token.kind != "word": + raise QueryError(f"Expected a value list, got {token.text!r}.") + return [v.strip() for v in _unquote(token.text).split(",") if v.strip()] + + +def _combine(left: Predicate, right: Predicate, op: Any) -> Predicate: + return lambda cols: op(left(cols), right(cols)) + + +def _column(cols: Dict[str, List[str]], name: str) -> np.ndarray: + if name not in cols: + raise QueryError( + f"Column {name!r} not found. Available: {', '.join(sorted(cols))}" + ) + return np.asarray(cols[name], dtype=str) + + +def _as_float(values: np.ndarray, column: str) -> np.ndarray: + """Parse a string column as floats, treating unparseable entries as NaN.""" + out = np.full(len(values), np.nan, dtype=np.float64) + for i, v in enumerate(values): + try: + out[i] = float(v) + except (TypeError, ValueError): + continue + return out + + +def _comparison(column: str, op: str, value: str) -> Predicate: + def run(cols: Dict[str, List[str]]) -> np.ndarray: + values = _column(cols, column) + if op == "==": + return values == value + if op == "!=": + return values != value + + try: + threshold = float(value) + except ValueError as exc: + raise QueryError( + f"Operator {op!r} needs a number, but got {value!r}." + ) from exc + + numeric = _as_float(values, column) + with np.errstate(invalid="ignore"): + if op == "<": + return numeric < threshold + if op == "<=": + return numeric <= threshold + if op == ">": + return numeric > threshold + return numeric >= threshold + + return run + + +def _membership(column: str, options: Sequence[str], negate: bool) -> Predicate: + wanted = set(options) + + def run(cols: Dict[str, List[str]]) -> np.ndarray: + values = _column(cols, column) + mask = np.isin(values, list(wanted)) + return np.logical_not(mask) if negate else mask + + return run + + +def compile_query(expr: str) -> Predicate: + """Compile a query string into a predicate over a chunk of columns.""" + tokens = tokenize(expr) + if not tokens: + raise QueryError("Query is empty.") + return _Parser(tokens).parse() + + +def referenced_columns(expr: str) -> List[str]: + """Names that look like column references, so only those need reading.""" + tokens = tokenize(expr) + names: List[str] = [] + for i, token in enumerate(tokens): + if token.kind != "word": + continue + following = tokens[i + 1] if i + 1 < len(tokens) else None + if following is not None and following.kind in ("op", "in", "not"): + name = _unquote(token.text) + if name not in names: + names.append(name) + return names diff --git a/src/adata/core/select.py b/src/adata/core/select.py new file mode 100644 index 0000000..f1049bc --- /dev/null +++ b/src/adata/core/select.py @@ -0,0 +1,122 @@ +"""Selecting rows of an axis by column value, without loading the store. + +Both `subset --query` and `split --by` need the same thing: walk an axis's +annotation columns in chunks and decide which rows to keep. Only the columns a +query actually mentions are read. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np + +from adata.core.info import get_axis_group +from adata.core.query import QueryError, compile_query, referenced_columns +from adata.elements.read import col_chunk_as_strings, dataframe_columns + + +def _available_columns(group: Any, index_name: str) -> List[str]: + cols = dataframe_columns(group, index_name) + return cols + [index_name] + + +def _read_chunk( + group: Any, + columns: List[str], start: int, end: int, cache: Dict +) -> Dict[str, List[str]]: + return { + name: col_chunk_as_strings(group, name, start, end, cache) + for name in columns + } + + +def select_indices( + root: Any, + axis: str, + expr: str, + *, + chunk_size: int = 100_000, +) -> np.ndarray: + """Row indices of `axis` matching the query `expr`. + + Only the columns the query references are read, so filtering on one + annotation does not touch the rest of the frame. + """ + group, n_rows, index_name = get_axis_group(root, axis) + predicate = compile_query(expr) + + wanted = referenced_columns(expr) + available = _available_columns(group, index_name) + missing = [c for c in wanted if c not in available] + if missing: + raise QueryError( + f"Column(s) {', '.join(missing)} not found in '{axis}'. " + f"Available: {', '.join(sorted(available))}" + ) + + cache: Dict = {} + kept: List[np.ndarray] = [] + for start in range(0, n_rows, chunk_size): + end = min(start + chunk_size, n_rows) + mask = np.asarray( + predicate(_read_chunk(group, wanted, start, end, cache)), dtype=bool + ) + if mask.any(): + kept.append(np.nonzero(mask)[0] + start) + + if not kept: + return np.empty(0, dtype=np.int64) + return np.concatenate(kept).astype(np.int64) + + +def group_indices( + root: Any, + axis: str, + column: str, + *, + chunk_size: int = 100_000, +) -> Tuple[Dict[str, np.ndarray], List[str]]: + """Group `axis` row indices by the value of `column`. + + Returns the groups and the distinct labels in order of first appearance, + so output ordering follows the data rather than the hash of the labels. + """ + group, n_rows, index_name = get_axis_group(root, axis) + + available = _available_columns(group, index_name) + if column not in available: + raise KeyError( + f"Column {column!r} not found in '{axis}'. " + f"Available: {', '.join(sorted(available))}" + ) + + cache: Dict = {} + buckets: Dict[str, List[np.ndarray]] = {} + order: List[str] = [] + + for start in range(0, n_rows, chunk_size): + end = min(start + chunk_size, n_rows) + values = np.asarray( + col_chunk_as_strings(group, column, start, end, cache), dtype=str + ) + for label in dict.fromkeys(values.tolist()): + if label not in buckets: + buckets[label] = [] + order.append(label) + buckets[label].append(np.nonzero(values == label)[0] + start) + + return ( + {k: np.concatenate(v).astype(np.int64) for k, v in buckets.items()}, + order, + ) + + +def read_names(root: Any, axis: str, indices: Optional[np.ndarray] = None) -> List[str]: + """Read an axis's index labels, optionally only at `indices`.""" + group, n_rows, index_name = get_axis_group(root, axis) + cache: Dict = {} + names = col_chunk_as_strings(group, index_name, 0, n_rows, cache) + if indices is None: + return names + return [names[i] for i in indices] diff --git a/src/adata/core/subset.py b/src/adata/core/subset.py index 1dba540..5e85e82 100644 --- a/src/adata/core/subset.py +++ b/src/adata/core/subset.py @@ -19,6 +19,7 @@ from adata.elements import spec from adata.elements.write import set_shape_attr, write_mapping +from adata.core.select import read_names, select_indices from adata.elements.read import ( decode_str_array, element_len, @@ -469,6 +470,42 @@ def subset_raw_group( copy_tree(src_raw[key], raw_dst, key) +def _select_axis( + src: Any, + axis: str, + name_file: Optional[Path], + query: Optional[str], + console: Console, +) -> Optional[np.ndarray]: + """Resolve an axis selection from a name list or a query, or None for all.""" + if name_file is None and query is None: + return None + + if name_file is not None and query is not None: + raise ValueError(f"Give either --{axis} or --{axis}-query, not both.") + + if query is not None: + console.print(f"[cyan]Evaluating {axis} query...[/]") + indices = select_indices(src, axis, query) + console.print(f"[green]Selected {len(indices)} {axis}[/]") + if len(indices) == 0: + raise ValueError(f"The {axis} query matched no rows.") + return indices + + keep = _read_name_file(name_file) + console.print(f"[cyan]Found {len(keep)} {axis} names to keep[/]") + names_ds, _ = resolve_index(src[axis], axis) + indices, missing = indices_from_name_set(names_ds, keep) + if missing: + console.print( + f"[yellow]Warning: {len(missing)} {axis} names not found in file[/]" + ) + console.print( + f"[green]Selected {len(indices)} {axis} (of {element_len(names_ds)})[/]" + ) + return indices + + def subset_h5ad( file: Path, output: Optional[Path], @@ -478,18 +515,23 @@ def subset_h5ad( chunk_rows: int = 1024, console: Console, inplace: bool = False, + obs_query: Optional[str] = None, + var_query: Optional[str] = None, + obs_indices: Optional[np.ndarray] = None, + var_indices: Optional[np.ndarray] = None, + zarr_format: Optional[int] = None, + quiet: bool = False, ) -> None: - obs_keep: Optional[Set[str]] = None - if obs_file is not None: - obs_keep = _read_name_file(obs_file) - console.print(f"[cyan]Found {len(obs_keep)} obs names to keep[/]") + """Write a copy of `file` narrowed to the selected obs and/or var. - var_keep: Optional[Set[str]] = None - if var_file is not None: - var_keep = _read_name_file(var_file) - console.print(f"[cyan]Found {len(var_keep)} var names to keep[/]") - - if obs_keep is None and var_keep is None: + Selection comes from a name file, a query, or indices computed by a caller + such as `split`. Exactly one source per axis. + """ + has_selection = any( + x is not None + for x in (obs_file, var_file, obs_query, var_query, obs_indices, var_indices) + ) + if not has_selection: raise ValueError("At least one of --obs or --var must be provided.") if not inplace and output is None: @@ -508,40 +550,33 @@ def subset_h5ad( else: dst_path = output + if zarr_format is None and detect_backend(file) == "zarr": + with open_store(file, "r") as probe: + zarr_format = probe.zarr_format + with console.status("[magenta]Opening files...[/]"): - with open_store(file, "r") as src_store, open_store(dst_path, "w") as dst_store: + with open_store(file, "r") as src_store, open_store( + dst_path, "w", zarr_format=zarr_format + ) as dst_store: src = src_store.root dst = dst_store.root - obs_idx = None - if obs_keep is not None: - console.print("[cyan]Matching obs names...[/]") - obs_group = src["obs"] - obs_names_ds, _ = resolve_index(obs_group, "obs") - - obs_idx, missing_obs = indices_from_name_set(obs_names_ds, obs_keep) - if missing_obs: - console.print( - f"[yellow]Warning: {len(missing_obs)} obs names not found in file[/]" - ) - console.print( - f"[green]Selected {len(obs_idx)} obs (of {element_len(obs_names_ds)})[/]" - ) - - var_idx = None - if var_keep is not None: - console.print("[cyan]Matching var names...[/]") - var_group = src["var"] - var_names_ds, _ = resolve_index(var_group, "var") + obs_idx = ( + obs_indices + if obs_indices is not None + else _select_axis(src, "obs", obs_file, obs_query, console) + ) + var_idx = ( + var_indices + if var_indices is not None + else _select_axis(src, "var", var_file, var_query, console) + ) - var_idx, missing_var = indices_from_name_set(var_names_ds, var_keep) - if missing_var: - console.print( - f"[yellow]Warning: {len(missing_var)} var names not found in file[/]" - ) - console.print( - f"[green]Selected {len(var_idx)} var (of {element_len(var_names_ds)})[/]" - ) + # raw/ has its own var axis, so it is matched by name rather than + # by reusing these indices. + var_keep: Optional[Set[str]] = None + if var_idx is not None and "var" in src: + var_keep = set(read_names(src, "var", var_idx)) tasks: List[str] = [] if "obs" in src: diff --git a/src/adata/formats/dataframe.py b/src/adata/formats/dataframe.py index 93110b7..5d5f466 100644 --- a/src/adata/formats/dataframe.py +++ b/src/adata/formats/dataframe.py @@ -11,10 +11,12 @@ from adata.core.read import col_chunk_as_strings from adata.formats.common import _resolve +from adata.util.path import norm_path from adata.elements.read import dataframe_columns, element_len, resolve_index from adata.formats.validate import validate_dimensions from adata.elements.write import ( write_categorical, + write_mapping, write_dataframe_header, write_dense, write_string_array, @@ -185,26 +187,26 @@ def import_dataframe( categorical: Optional[List[str]] = None, auto_categorical: bool = True, ) -> None: - """Replace `obs` or `var` with the contents of a CSV file. + """Replace a dataframe at `obj` with the contents of a CSV file. - Columns that parse cleanly as integers or floats become numeric arrays; - the rest become either a `categorical` or a `string-array`. `categorical` - names columns to force, and `auto_categorical` applies the heuristic to - the remainder. + `obj` is any dataframe path, not only "obs" or "var". Columns that parse + cleanly as integers or floats become numeric arrays; the rest become + either a `categorical` or a `string-array`. `categorical` names columns to + force, and `auto_categorical` applies the heuristic to the remainder. """ - if obj not in ("obs", "var"): - raise ValueError( - f"CSV import is only supported for 'obs' or 'var', not '{obj}'." - ) - + obj = norm_path(obj) rows, data_columns, index_values, _ = _read_csv(input_file, index_column) n_rows = len(rows) validate_dimensions(root, obj, (n_rows,), console) - index_name = "_index" + parts = obj.split("/") + parent = root + for part in parts[:-1]: + parent = parent[part] if part in parent else write_mapping(parent, part) + group = write_dataframe_header( - root, obj, index_values, data_columns, index_name=index_name + parent, parts[-1], index_values, data_columns, index_name="_index" ) forced = set(categorical or ()) diff --git a/src/adata/formats/image.py b/src/adata/formats/image.py index e173325..779f2bc 100644 --- a/src/adata/formats/image.py +++ b/src/adata/formats/image.py @@ -7,8 +7,10 @@ from PIL import Image from rich.console import Console +from adata.elements.write import write_dense, write_mapping from adata.formats.common import _resolve from adata.storage import is_dataset +from adata.util.path import norm_path def export_image(root: Any, obj: str, out: Path, console: Console) -> None: @@ -45,3 +47,27 @@ def export_image(root: Any, obj: str, out: Path, console: Console) -> None: out.parent.mkdir(parents=True, exist_ok=True) img.save(out) console.print(f"[green]Wrote[/] {out}") + + +def import_image(root: Any, obj: str, input_file: Path, console: Console) -> None: + """Read an image file into the store as a dense array. + + Kept as raw pixels (H, W) or (H, W, C) with no encoding beyond `array`, + which is how spatial tooling stores tissue images under uns/spatial. + """ + obj = norm_path(obj) + arr = np.asarray(Image.open(input_file)) + + if arr.ndim not in (2, 3): + raise ValueError(f"Expected a 2D or 3D image; got shape {arr.shape}.") + + parts = obj.split("/") + parent = root + for part in parts[:-1]: + parent = parent[part] if part in parent else write_mapping(parent, part) + + write_dense(parent, parts[-1], arr, replace=True) + console.print( + f"[green]Imported[/] {'x'.join(str(d) for d in arr.shape)} " + f"image ({arr.dtype}) into '{obj}'" + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9bbb49d..97e0d1a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -511,7 +511,7 @@ def test_subset_command_no_filters(self, sample_h5ad_file, temp_dir): assert result.exit_code == 1 # Check both stdout and stderr since Console uses stderr=True output_text = result.stdout + result.stderr - assert "At least one of --obs or --var must be provided" in output_text + assert "--obs-query" in output_text def test_subset_command_chunk_rows(self, sample_h5ad_file, temp_dir): """Test subset command with custom chunk size.""" diff --git a/tests/test_commands_phase2.py b/tests/test_commands_phase2.py new file mode 100644 index 0000000..d67bd62 --- /dev/null +++ b/tests/test_commands_phase2.py @@ -0,0 +1,452 @@ +"""Tests for create, split, query-based subset and concat. + +These lean on anndata to build inputs and check outputs, for the same reason +as tests/test_anndata_roundtrip.py: the point is interoperability, not +self-consistency. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import numpy as np +import pytest +from typer.testing import CliRunner + +from adata.cli import app + +ad = pytest.importorskip("anndata", reason="anndata is required for these tests") +pd = pytest.importorskip("pandas") +sparse = pytest.importorskip("scipy.sparse") + +runner = CliRunner() + + +_ANSI = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]") + + +def _out(result) -> str: + """Merged stdout+stderr with Rich markup and its line wrapping removed. + + Rich wraps long paths across lines, so a message can be split mid-sentence; + collapsing whitespace lets assertions match the message as written. + """ + text = result.stdout + (result.stderr or "") + return " ".join(_ANSI.sub("", text).split()) + + +def _make( + path: Path, + cells: list, + genes: list, + batch: str = "b", + extra_col: str | None = None, + seed: int = 0, +) -> Path: + rng = np.random.default_rng(seed) + X = sparse.csr_matrix(rng.poisson(1.0, (len(cells), len(genes))).astype("float32")) + obs = pd.DataFrame( + { + "batch": pd.Categorical([batch] * len(cells)), + "score": np.arange(len(cells), dtype="float32"), + }, + index=cells, + ) + if extra_col: + obs[extra_col] = np.arange(len(cells), dtype="int32") + var = pd.DataFrame({"gene_type": ["protein"] * len(genes)}, index=genes) + obj = ad.AnnData(X=X, obs=obs, var=var) + obj.obsm["X_umap"] = rng.normal(size=(len(cells), 2)).astype("float32") + obj.uns["shared"] = "same-everywhere" + obj.uns["only_here"] = batch + obj.write_h5ad(path) + return path + + +# --------------------------------------------------------------------------- +# create + + +@pytest.mark.parametrize("fmt", ["h5ad", "zarr"]) +def test_create_makes_a_valid_empty_store(tmp_path, fmt): + out = tmp_path / f"new.{fmt}" + result = runner.invoke( + app, ["create", str(out), "--n-obs", "5", "--n-var", "3"] + ) + assert result.exit_code == 0, _out(result) + + obj = ad.read_zarr(out) if fmt == "zarr" else ad.read_h5ad(out) + assert obj.shape == (5, 3) + assert list(obj.obs_names) == [f"cell_{i}" for i in range(5)] + + +def test_create_takes_names_from_files(tmp_path): + (tmp_path / "cells.txt").write_text("c1\nc2\n") + (tmp_path / "genes.txt").write_text("g1\ng2\ng3\n") + out = tmp_path / "named.h5ad" + + result = runner.invoke( + app, + ["create", str(out), + "--obs-names", str(tmp_path / "cells.txt"), + "--var-names", str(tmp_path / "genes.txt")], + ) + assert result.exit_code == 0, _out(result) + + obj = ad.read_h5ad(out) + assert list(obj.obs_names) == ["c1", "c2"] + assert list(obj.var_names) == ["g1", "g2", "g3"] + + +def test_create_refuses_to_clobber_without_force(tmp_path): + out = tmp_path / "exists.h5ad" + assert runner.invoke(app, ["create", str(out), "--n-obs", "2", "--n-var", "2"]).exit_code == 0 + result = runner.invoke(app, ["create", str(out), "--n-obs", "3", "--n-var", "2"]) + assert result.exit_code == 1 + assert "--force" in _out(result) + + +def test_create_rejects_conflicting_axis_size(tmp_path): + (tmp_path / "cells.txt").write_text("c1\nc2\n") + result = runner.invoke( + app, + ["create", str(tmp_path / "x.h5ad"), "--n-obs", "5", + "--obs-names", str(tmp_path / "cells.txt"), "--n-var", "2"], + ) + assert result.exit_code == 1 + assert "has 2 names" in _out(result) + + +def test_create_then_import_builds_a_usable_object(tmp_path): + """The workflow the create command exists to enable.""" + out = tmp_path / "built.h5ad" + np.save(tmp_path / "X.npy", np.arange(6, dtype="float32").reshape(3, 2)) + (tmp_path / "meta.csv").write_text("_index,sample\nc1,S1\nc2,S2\nc3,S1\n") + (tmp_path / "cells.txt").write_text("c1\nc2\nc3\n") + (tmp_path / "genes.txt").write_text("g1\ng2\n") + + steps = [ + ["create", str(out), + "--obs-names", str(tmp_path / "cells.txt"), + "--var-names", str(tmp_path / "genes.txt")], + ["import", "array", str(out), "X", str(tmp_path / "X.npy"), "--inplace"], + ["import", "dataframe", str(out), "obs", str(tmp_path / "meta.csv"), + "--inplace", "-i", "_index"], + ] + for step in steps: + result = runner.invoke(app, step) + assert result.exit_code == 0, _out(result) + + obj = ad.read_h5ad(out) + assert obj.shape == (3, 2) + assert list(obj.obs["sample"]) == ["S1", "S2", "S1"] + assert np.array_equal(obj.X, np.arange(6, dtype="float32").reshape(3, 2)) + + +# --------------------------------------------------------------------------- +# subset --query + + +@pytest.fixture +def sample(tmp_path) -> Path: + return _make(tmp_path / "sample.h5ad", ["c1", "c2", "c3", "c4"], ["g1", "g2"]) + + +def test_subset_by_query(tmp_path, sample): + out = tmp_path / "q.h5ad" + result = runner.invoke( + app, ["subset", str(sample), "-o", str(out), "-q", "score > 1"] + ) + assert result.exit_code == 0, _out(result) + assert list(ad.read_h5ad(out).obs_names) == ["c3", "c4"] + + +def test_subset_query_combines_conditions(tmp_path, sample): + out = tmp_path / "q2.h5ad" + result = runner.invoke( + app, + ["subset", str(sample), "-o", str(out), + "-q", "score >= 1 and score < 3"], + ) + assert result.exit_code == 0, _out(result) + assert list(ad.read_h5ad(out).obs_names) == ["c2", "c3"] + + +def test_subset_var_query(tmp_path, sample): + out = tmp_path / "qv.h5ad" + result = runner.invoke( + app, + ["subset", str(sample), "-o", str(out), "--var-query", "gene_type == protein"], + ) + assert result.exit_code == 0, _out(result) + assert list(ad.read_h5ad(out).var_names) == ["g1", "g2"] + + +def test_subset_query_matching_nothing_is_an_error(tmp_path, sample): + result = runner.invoke( + app, + ["subset", str(sample), "-o", str(tmp_path / "e.h5ad"), "-q", "score > 999"], + ) + assert result.exit_code == 1 + assert "matched no rows" in _out(result) + + +def test_subset_query_on_unknown_column_is_reported(tmp_path, sample): + result = runner.invoke( + app, + ["subset", str(sample), "-o", str(tmp_path / "e.h5ad"), "-q", "nope == 1"], + ) + assert result.exit_code == 1 + assert "not found" in _out(result) + + +def test_subset_requires_some_selection(tmp_path, sample): + result = runner.invoke(app, ["subset", str(sample), "-o", str(tmp_path / "e.h5ad")]) + assert result.exit_code == 1 + assert "--obs-query" in _out(result) + + +# --------------------------------------------------------------------------- +# split + + +def test_split_writes_one_store_per_value(tmp_path): + src = tmp_path / "many.h5ad" + rng = np.random.default_rng(0) + obs = pd.DataFrame( + {"group": pd.Categorical(["x", "y", "x", "z", "y", "x"])}, + index=[f"c{i}" for i in range(6)], + ) + obj = ad.AnnData( + X=sparse.csr_matrix(rng.poisson(1.0, (6, 3)).astype("float32")), + obs=obs, + var=pd.DataFrame(index=["g1", "g2", "g3"]), + ) + obj.write_h5ad(src) + + out_dir = tmp_path / "split" + result = runner.invoke( + app, ["split", str(src), "--by", "group", "-o", str(out_dir)] + ) + assert result.exit_code == 0, _out(result) + + assert {p.name for p in out_dir.glob("*.h5ad")} == {"x.h5ad", "y.h5ad", "z.h5ad"} + assert list(ad.read_h5ad(out_dir / "x.h5ad").obs_names) == ["c0", "c2", "c5"] + assert ad.read_h5ad(out_dir / "z.h5ad").shape == (1, 3) + + manifest = (out_dir / "many_manifest.csv").read_text().splitlines() + assert manifest[0] == "id,column,value,anndatas,n" + assert len(manifest) == 4 + + +def test_split_dry_run_writes_nothing(tmp_path, sample): + out_dir = tmp_path / "dry" + result = runner.invoke( + app, ["split", str(sample), "--by", "batch", "-o", str(out_dir), "--dry-run"] + ) + assert result.exit_code == 0, _out(result) + assert not out_dir.exists() + + +def test_split_min_size_skips_small_groups(tmp_path): + src = tmp_path / "skew.h5ad" + obs = pd.DataFrame( + {"group": pd.Categorical(["big", "big", "big", "tiny"])}, + index=[f"c{i}" for i in range(4)], + ) + ad.AnnData( + X=np.ones((4, 2), dtype="float32"), + obs=obs, + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(src) + + out_dir = tmp_path / "split" + result = runner.invoke( + app, + ["split", str(src), "--by", "group", "-o", str(out_dir), "--min-size", "2"], + ) + assert result.exit_code == 0, _out(result) + assert {p.name for p in out_dir.glob("*.h5ad")} == {"big.h5ad"} + assert "Skipping" in _out(result) + + +def test_split_sanitises_labels_for_filenames(tmp_path): + src = tmp_path / "messy.h5ad" + obs = pd.DataFrame( + {"group": ["a/b", "c d"]}, index=["c1", "c2"] + ) + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=obs, + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(src) + + out_dir = tmp_path / "split" + assert runner.invoke( + app, ["split", str(src), "--by", "group", "-o", str(out_dir)] + ).exit_code == 0 + assert {p.name for p in out_dir.glob("*.h5ad")} == {"a_b.h5ad", "c_d.h5ad"} + + +def test_split_unknown_column_is_reported(tmp_path, sample): + result = runner.invoke( + app, ["split", str(sample), "--by", "nope", "-o", str(tmp_path / "o")] + ) + assert result.exit_code == 1 + assert "not found" in _out(result) + + +# --------------------------------------------------------------------------- +# concat + + +def test_split_then_concat_reconstructs_the_original(tmp_path): + """The strongest end-to-end check available: the two must be inverses.""" + rng = np.random.default_rng(3) + cells = [f"c{i}" for i in range(6)] + obs = pd.DataFrame( + { + "group": pd.Categorical(["x", "y", "x", "z", "y", "x"]), + "score": np.arange(6, dtype="float32"), + "nullable": pd.array([1, 2, None, 4, 5, None], dtype="Int32"), + }, + index=cells, + ) + original = ad.AnnData( + X=sparse.csr_matrix(rng.poisson(1.0, (6, 3)).astype("float32")), + obs=obs, + var=pd.DataFrame(index=["g1", "g2", "g3"]), + ) + original.obsm["X_umap"] = rng.normal(size=(6, 2)).astype("float32") + src = tmp_path / "src.h5ad" + original.write_h5ad(src) + + out_dir = tmp_path / "parts" + assert runner.invoke( + app, ["split", str(src), "--by", "group", "-o", str(out_dir)] + ).exit_code == 0 + + parts = sorted(str(p) for p in out_dir.glob("*.h5ad")) + assert len(parts) == 3 + + merged = tmp_path / "merged.h5ad" + result = runner.invoke(app, ["concat", *parts, "-o", str(merged)]) + assert result.exit_code == 0, _out(result) + + got = ad.read_h5ad(merged) + assert got.shape == original.shape + assert sorted(got.obs_names) == sorted(original.obs_names) + + # Reorder the original to the concatenated order before comparing. + ref = original[list(got.obs_names)] + assert abs(got.X - ref.X).nnz == 0 + assert np.allclose(got.obsm["X_umap"], ref.obsm["X_umap"]) + assert list(got.obs["group"]) == list(ref.obs["group"]) + assert str(got.obs["group"].dtype) == "category" + assert got.obs["nullable"].tolist() == ref.obs["nullable"].tolist() + + +@pytest.mark.parametrize("join", ["inner", "outer"]) +def test_concat_matches_anndata(tmp_path, join): + """Our output must agree with anndata.concat on the same inputs.""" + a = _make(tmp_path / "a.h5ad", ["c1", "c2"], ["g1", "g2", "g3"], batch="A", seed=1) + b = _make(tmp_path / "b.h5ad", ["c1", "c3"], ["g2", "g3", "g4"], batch="B", seed=2) + out = tmp_path / f"m_{join}.h5ad" + + result = runner.invoke( + app, + ["concat", str(a), str(b), "-o", str(out), "--join", join, + "--label", "batch_id", "--keys", "A,B", "--index-unique", "-"], + ) + assert result.exit_code == 0, _out(result) + + ours = ad.read_h5ad(out) + theirs = ad.concat( + [ad.read_h5ad(a), ad.read_h5ad(b)], + join=join, label="batch_id", keys=["A", "B"], index_unique="-", + ) + + assert list(ours.var_names) == list(theirs.var_names) + assert list(ours.obs_names) == list(theirs.obs_names) + assert np.array_equal( + np.asarray(ours.X.todense()), np.asarray(theirs.X.todense()) + ) + assert list(ours.obs["batch_id"]) == list(theirs.obs["batch_id"]) + + +def test_concat_preserves_column_dtypes(tmp_path): + a = _make(tmp_path / "a.h5ad", ["c1", "c2"], ["g1", "g2"], batch="A", seed=1) + b = _make(tmp_path / "b.h5ad", ["c3", "c4"], ["g1", "g2"], batch="B", seed=2) + out = tmp_path / "m.h5ad" + + assert runner.invoke(app, ["concat", str(a), str(b), "-o", str(out)]).exit_code == 0 + + obj = ad.read_h5ad(out) + assert str(obj.obs["batch"].dtype) == "category" + assert sorted(obj.obs["batch"].cat.categories) == ["A", "B"] + assert str(obj.obs["score"].dtype) == "float32" + assert "X_umap" in obj.obsm + + +def test_concat_uns_merge_keeps_only_agreeing_values(tmp_path): + a = _make(tmp_path / "a.h5ad", ["c1"], ["g1", "g2"], batch="A", seed=1) + b = _make(tmp_path / "b.h5ad", ["c2"], ["g1", "g2"], batch="B", seed=2) + out = tmp_path / "m.h5ad" + + result = runner.invoke( + app, ["concat", str(a), str(b), "-o", str(out), "--uns-merge", "same"] + ) + assert result.exit_code == 0, _out(result) + + uns = ad.read_h5ad(out).uns + assert uns["shared"] == "same-everywhere" + assert "only_here" not in uns, "values that differ must not survive 'same'" + + +def test_concat_drops_uns_by_default(tmp_path): + a = _make(tmp_path / "a.h5ad", ["c1"], ["g1"], batch="A") + b = _make(tmp_path / "b.h5ad", ["c2"], ["g1"], batch="B") + out = tmp_path / "m.h5ad" + + assert runner.invoke(app, ["concat", str(a), str(b), "-o", str(out)]).exit_code == 0 + assert dict(ad.read_h5ad(out).uns) == {} + + +def test_concat_warns_about_duplicate_obs_names(tmp_path): + a = _make(tmp_path / "a.h5ad", ["c1"], ["g1"], batch="A") + b = _make(tmp_path / "b.h5ad", ["c1"], ["g1"], batch="B") + result = runner.invoke( + app, ["concat", str(a), str(b), "-o", str(tmp_path / "m.h5ad")] + ) + assert result.exit_code == 0, _out(result) + assert "not unique" in _out(result) + + +def test_concat_rejects_disjoint_vars_on_inner_join(tmp_path): + a = _make(tmp_path / "a.h5ad", ["c1"], ["g1", "g2"], batch="A") + b = _make(tmp_path / "b.h5ad", ["c2"], ["g3", "g4"], batch="B") + result = runner.invoke( + app, ["concat", str(a), str(b), "-o", str(tmp_path / "m.h5ad")] + ) + assert result.exit_code == 1 + assert "--join outer" in _out(result) + + +def test_concat_needs_two_inputs(tmp_path): + a = _make(tmp_path / "a.h5ad", ["c1"], ["g1"]) + result = runner.invoke(app, ["concat", str(a), "-o", str(tmp_path / "m.h5ad")]) + assert result.exit_code == 1 + assert "at least two" in _out(result) + + +def test_concat_rejects_an_unknown_merge_strategy(tmp_path): + a = _make(tmp_path / "a.h5ad", ["c1"], ["g1"]) + b = _make(tmp_path / "b.h5ad", ["c2"], ["g1"]) + result = runner.invoke( + app, + ["concat", str(a), str(b), "-o", str(tmp_path / "m.h5ad"), + "--uns-merge", "bogus"], + ) + assert result.exit_code == 1 + assert "must be one of" in _out(result) diff --git a/tests/test_import.py b/tests/test_import.py index b15fc9c..c2958ae 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -166,8 +166,8 @@ def test_import_dataframe_invalid_index_column(self, sample_h5ad_file, temp_dir) assert result.exit_code == 1 assert "not found" in result.output.lower() - def test_import_dataframe_not_obs_var(self, sample_h5ad_file, temp_dir): - """Test that dataframe import is only allowed for obs/var.""" + def test_import_dataframe_at_arbitrary_path(self, sample_h5ad_file, temp_dir): + """Dataframes can be imported anywhere, not just obs/var (issue #4).""" csv_file = temp_dir / "data.csv" csv_file.write_text("a,b\n1,2\n") @@ -182,8 +182,11 @@ def test_import_dataframe_not_obs_var(self, sample_h5ad_file, temp_dir): "--inplace", ], ) - assert result.exit_code == 1 - assert "obs" in result.output or "var" in result.output + assert result.exit_code == 0, result.output + + with h5py.File(sample_h5ad_file, "r") as f: + assert f["uns/data"].attrs["encoding-type"] == "dataframe" + assert "b" in f["uns/data"] def test_import_dataframe_requires_output_or_inplace( self, sample_h5ad_file, temp_dir diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..87008e6 --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,83 @@ +"""Tests for the obs/var predicate language.""" + +from __future__ import annotations + +import pytest + +from adata.core.query import QueryError, compile_query, referenced_columns + +COLUMNS = { + "cluster": ["A", "B", "A", "C"], + "n_counts": ["10", "200", "30", "4000"], + "flag": ["True", "False", "True", "False"], +} + + +@pytest.mark.parametrize( + "expr,expected", + [ + ("cluster == A", [True, False, True, False]), + ("cluster != A", [False, True, False, True]), + ("cluster in A,B", [True, True, True, False]), + ("cluster not in A", [False, True, False, True]), + ("n_counts > 100", [False, True, False, True]), + ("n_counts >= 30", [False, True, True, True]), + ("n_counts < 30", [True, False, False, False]), + ("n_counts <= 30", [True, False, True, False]), + ("flag == True", [True, False, True, False]), + ("cluster == A and n_counts > 20", [False, False, True, False]), + ("cluster == C or n_counts < 20", [True, False, False, True]), + ("not cluster == A", [False, True, False, True]), + ("(cluster == A or cluster == C) and n_counts > 20", [False, False, True, True]), + ], +) +def test_predicates(expr, expected): + assert compile_query(expr)(COLUMNS).tolist() == expected + + +def test_quoted_values_allow_spaces(): + columns = {"label": ["cell type A", "other"]} + assert compile_query('label == "cell type A"')(columns).tolist() == [True, False] + + +def test_parenthesised_value_list(): + assert compile_query("cluster in (A, C)")(COLUMNS).tolist() == [ + True, + False, + True, + True, + ] + + +def test_referenced_columns_ignores_values(): + assert referenced_columns("cluster == A and n_counts > 5") == [ + "cluster", + "n_counts", + ] + assert referenced_columns("cluster in A,B") == ["cluster"] + + +def test_unknown_column_is_reported(): + with pytest.raises(QueryError, match="nope"): + compile_query("nope == 1")(COLUMNS) + + +def test_ordering_operator_needs_a_number(): + with pytest.raises(QueryError, match="needs a number"): + compile_query("cluster > abc")(COLUMNS) + + +def test_unparseable_numbers_do_not_match(): + """A non-numeric entry is simply excluded rather than raising.""" + columns = {"x": ["1", "not-a-number", "3"]} + assert compile_query("x > 0")(columns).tolist() == [True, False, True] + + +def test_empty_query_rejected(): + with pytest.raises(QueryError): + compile_query(" ") + + +def test_unbalanced_parentheses_rejected(): + with pytest.raises(QueryError): + compile_query("(cluster == A") From 5d189b60d482147d55f86c59445dfc4938df7c28 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 12:29:19 +0100 Subject: [PATCH 12/23] Publish to PyPI, add a docs site, and swap csvkit for duckdb Prepares the first PyPI release and gives the project documentation that lives somewhere other than the repo root. Packaging: `Documentation` was "README.md", which is not a URL and PyPI rejects it. Adds license-files, Operating System and Typing classifiers, Python 3.13, py.typed, and a Changelog URL. `__version__` reads from installed metadata rather than being duplicated, and `adata --version` reports it. publish.yml publishes on a tag using PyPI trusted publishing, so there is no API token to hold. It reuses tests.yml via workflow_call, so a release cannot skip the suite, and refuses to publish when the tag does not match the version in pyproject.toml -- nothing enforced that before, and 0.3.2 was tagged against a tree declaring 0.3.1. A workflow_dispatch path targets TestPyPI first. Docs become a GitHub Pages site served from docs/ with plain Jekyll, so the files stay readable as markdown on github.com. Adds index.md, a full COMMANDS.md reference, and to each ELEMENTS doc a section on what this tool actually does with each element -- they were pure spec transcriptions saying nothing about the CLI. Documents the `null` encoding (absent from the upstream prose spec but written by anndata 0.12+) and, for Zarr, the v2/v3 differences that matter when copying between stores. csvkit is replaced by duckdb in the tutorial and the image. The tutorial now leads with `--obs-query`, which covers the example it used csvsql for, and keeps duckdb for filters that genuinely need SQL. The image drops the csvkit side-venv for a single static duckdb binary, and passes the --locked flag its comment already claimed. Verified: the wheel installs into a clean venv and runs; the image builds and the documented duckdb workflow runs inside it end to end. Co-Authored-By: Claude Opus 5 --- .github/workflows/publish.yml | 80 ++++++++++ .github/workflows/quay-on-tag.yml | 2 +- .github/workflows/tests.yml | 2 + CHANGELOG.md | 84 +++++++++++ Dockerfile | 49 ++++--- README.md | 15 +- docs/COMMANDS.md | 235 ++++++++++++++++++++++++++++++ docs/ELEMENTS_h5ad.md | 36 +++++ docs/ELEMENTS_zarr.md | 63 ++++++++ docs/GET_STARTED.md | 57 ++++++-- docs/_config.yml | 11 ++ docs/index.md | 71 +++++++++ pyproject.toml | 21 ++- src/adata/__init__.py | 10 ++ src/adata/cli.py | 27 +++- src/adata/py.typed | 0 uv.lock | 2 +- 17 files changed, 720 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 CHANGELOG.md create mode 100644 docs/COMMANDS.md create mode 100644 docs/_config.yml create mode 100644 docs/index.md create mode 100644 src/adata/py.typed diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..9f13d2c --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,80 @@ +name: Publish + +# Tags are bare MAJOR.MINOR.PATCH, matching the existing convention. +on: + push: + tags: ["*"] + workflow_dispatch: + inputs: + target: + description: Where to publish + required: true + default: testpypi + type: choice + options: [testpypi, pypi] + +jobs: + # The tag must match the version in pyproject.toml. Nothing enforced this + # before, and 0.3.2 was tagged against a tree declaring 0.3.1. + check-version: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + - name: Tag must match the declared version + run: | + declared=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2) + tag="${GITHUB_REF_NAME}" + echo "pyproject: $declared, tag: $tag" + if [ "$declared" != "$tag" ]; then + echo "::error::Tag $tag does not match pyproject version $declared" + exit 1 + fi + + test: + uses: ./.github/workflows/tests.yml + + build: + needs: [test] + if: always() && needs.test.result == 'success' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + - run: uv build + - name: Check metadata renders on PyPI + run: uvx twine check dist/* + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish-testpypi: + needs: [build] + if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi' + runs-on: ubuntu-latest + environment: testpypi + permissions: + id-token: write # trusted publishing; no API token secret needed + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + publish-pypi: + needs: [build, check-version] + if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi') + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/quay-on-tag.yml b/.github/workflows/quay-on-tag.yml index e0b9fdb..d33ea91 100644 --- a/.github/workflows/quay-on-tag.yml +++ b/.github/workflows/quay-on-tag.yml @@ -31,7 +31,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: quay.io/cellgeni/h5ad-cli + images: quay.io/cellgeni/adata-cli tags: | type=ref,event=tag # push "latest" only for tags WITHOUT hyphens (excludes pre-releases like 1.0.0-beta): diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f12a9d1..c7128d8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,6 +5,8 @@ on: branches: [main, dev] pull_request: branches: [main, dev] + # Reused by publish.yml, so a release cannot skip the suite. + workflow_call: concurrency: group: tests-${{ github.workflow }}-${{ github.ref }} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1732dfb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,84 @@ +# Changelog + +Notable changes to `adata-cli`. Versions are `MAJOR.MINOR.PATCH`; tags carry no +`v` prefix. + +## 0.5.0 + +Renamed from `h5ad` to `adata-cli`, restored compatibility with current +AnnData files, and added four commands. + +### Renamed + +- Distribution `adata-cli`, import package `adata`, command `adata`. +- `info` is now `view`. +- The `h5ad` command and the `info` subcommand remain as aliases that warn and + then run normally. **Both are removed in 1.0.0.** + +### Fixed + +- **The CLI could not read any file written by anndata >= 0.11.** Since pandas' + `future.infer_string` became the default, anndata writes `obs/_index` as a + `nullable-string-array` group rather than a dataset, and `axis_len` required + a dataset — so `info`, `export dataframe` and `subset` all failed outright. +- **HDF5 -> Zarr conversion failed on any store with string columns**, i.e. all + of them: an h5py variable-length string dataset reports `dtype == object`, + which Zarr rejects. All four backend pairings now convert. +- **`subset` silently dropped `raw/`.** It is now carried over and matched + against its own var axis. Unrecognised top-level keys are copied with a + warning rather than dropped. +- **`subset` corrupted group-valued columns**, copying them whole while + narrowing everything else. This was invisible before, because files + containing such columns could not be read at all. +- Categoricals no longer degrade to plain strings on a CSV round-trip. +- `None` round-trips via anndata's `null` encoding instead of an invented + `_is_none` marker attribute. +- `column-order` is honoured on export; previously columns came out in + whatever order the backend enumerated (alphabetical on HDF5). +- Type detection dispatches on `encoding-type` before falling back to + structure. A group merely *containing* a member named `obs_names` is no + longer misreported as a dataframe. +- `subset` prefers the declared `_index` over the `obs_names`/`var_names` + convention, which had been backwards. +- A Zarr v2 store is no longer silently upgraded to v3. +- Chunk shapes are clamped on the axis-column path, which could raise from + h5py when subsetting below a column's chunk size. + +### Added + +- `adata ls` — tree listing for any HDF5 or Zarr store, with no AnnData + assumptions, so `.loom` and plain `.h5` work. `--long`, `--depth`, and `-1` + for bare paths that pipe into other tools. +- `adata create` — write a new, empty store for `import` to fill in. +- `adata split --by ` — one store per distinct value, with a CSV + manifest. (#2) +- `adata concat` — concatenate along obs with `--join inner|outer`, + `--label`/`--keys`/`--index-unique` and merge strategies for var and uns. + Verified to agree with `anndata.concat`. +- `adata subset --obs-query` / `--var-query` — a small predicate language + (`==`, `!=`, `<`, `<=`, `>`, `>=`, `in`, `not in`, `and`, `or`, `not`, + parentheses) evaluated while streaming. No new dependency. +- `adata import image`, and `import` at any path rather than only `obs`/`var`. +- `--categorical` / `--no-auto-categorical` on `import dataframe`. +- `export dataframe` from any dataframe path, not only `obs`/`var`. (#4) +- `export array` can write to stdout. (#4) +- `adata --version`. +- `--zarr-format` on the commands that create stores. + +### Changed + +- Results go to stdout and status to stderr throughout. `view --tree` + previously wrote its tree to stderr and only the header to stdout. (#4) +- Everything written is tagged with its `encoding-type` and + `encoding-version`; text is always variable-length UTF-8, as the spec + requires, rather than fixed-width bytes. +- Sparse subsetting streams in blocks instead of loading `data`/`indices`/ + `indptr` whole. +- The Docker image ships `duckdb` in place of `csvkit`. +- CI runs the whole suite on Python 3.12 and 3.13 rather than naming test + files individually — which is why `test_storage_root_attrs.py` had never + run. + +## 0.3.2 and earlier + +See the git history. These releases were published to Quay only. diff --git a/Dockerfile b/Dockerfile index 1a0a2cc..5955f68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,40 @@ # Base image: Python 3.12 + uv preinstalled (Debian slim) FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim -# Environment variables ENV UV_NO_DEV=1 -ENV VENV=/env -# Work directory inside the container WORKDIR /cli -# Install git so we can clone the repo -RUN apt-get update \ - && apt-get install -y --no-install-recommends git \ - && rm -rf /var/lib/apt/lists/* - - -# Copy the project files (from GitHub Actions checkout context) +# Copy the project files (from the GitHub Actions checkout context) COPY . . +# --locked asserts that uv.lock is in sync with pyproject.toml, so an image +# can never be built from a lockfile that drifted. +RUN uv sync --locked -# Install the project according to pyproject.toml + uv.lock -# --locked asserts that uv.lock is in sync with pyproject.toml -RUN uv sync - -# Create separate venv for csvkit to avoid dependency conflicts -RUN uv venv $VENV --python 3.12 && \ - uv pip install --python $VENV/bin/python csvkit +# duckdb, for the filtering workflows in the docs: export obs to CSV, query it, +# feed the names back to `adata subset --obs`. A single static binary, so it +# needs no venv and cannot conflict with the project's dependencies. +ARG DUCKDB_VERSION=v1.1.3 +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl unzip \ + && ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in \ + amd64) DUCKDB_ARCH=amd64 ;; \ + arm64) DUCKDB_ARCH=aarch64 ;; \ + *) echo "unsupported architecture: $ARCH" >&2; exit 1 ;; \ + esac \ + && curl -fsSL -o /tmp/duckdb.zip \ + "https://github.com/duckdb/duckdb/releases/download/${DUCKDB_VERSION}/duckdb_cli-linux-${DUCKDB_ARCH}.zip" \ + && unzip -q /tmp/duckdb.zip -d /usr/local/bin \ + && chmod +x /usr/local/bin/duckdb \ + && rm /tmp/duckdb.zip \ + && apt-get purge -y curl unzip \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* -# Put the project venv on PATH so `h5ad` is directly runnable -ENV PATH="/cli/.venv/bin:${VENV}/bin:${PATH}" +# Put the project venv on PATH so `adata` is directly runnable +ENV PATH="/cli/.venv/bin:${PATH}" -# Default entrypoint: run the CLI -ENTRYPOINT ["h5ad"] +ENTRYPOINT ["adata"] CMD ["--help"] - diff --git a/README.md b/README.md index 05a1f8e..7fd0157 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,15 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with - Converts between HDF5 and Zarr (v2 and v3) in either direction - Rich terminal output with progress indicators, kept on stderr so results pipe cleanly +**Documentation: [cellgeni.github.io/adata-cli](https://cellgeni.github.io/adata-cli/)** + ## Installation -Using [uv](https://docs.astral.sh/uv/) (recommended): +```bash +pip install adata-cli +``` + +From source with [uv](https://docs.astral.sh/uv/): ```bash git clone https://github.com/cellgeni/adata-cli.git cd adata-cli @@ -69,7 +75,12 @@ adata split data.h5ad --by sample -o per_sample/ adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample ``` -See [docs/GET_STARTED.md](docs/GET_STARTED.md) for a short tutorial. +## Documentation + +- [Get started](docs/GET_STARTED.md) — a short tutorial +- [Command reference](docs/COMMANDS.md) — every command and flag +- [Element spec: HDF5](docs/ELEMENTS_h5ad.md) / [Zarr](docs/ELEMENTS_zarr.md) — the on-disk format, and what this tool does with it +- [Changelog](CHANGELOG.md) ## Docker diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md new file mode 100644 index 0000000..66caad8 --- /dev/null +++ b/docs/COMMANDS.md @@ -0,0 +1,235 @@ +# Command reference + +Every command takes a `.h5ad` file or a `.zarr` directory; the backend is +detected from the path. Results go to **stdout** and status to **stderr**, so +output pipes cleanly: + +```bash +adata export dataframe data.h5ad obs 2>/dev/null | head +adata ls data.h5ad -1 | grep spatial +``` + +Run `adata --help` for the authoritative flag list. + +## Table of contents + +- [view](#view) +- [ls](#ls) +- [subset](#subset) +- [split](#split) +- [concat](#concat) +- [create](#create) +- [export](#export) +- [import](#import) +- [Query expressions](#query-expressions) +- [Zarr format selection](#zarr-format-selection) + +--- + +## `view` + +AnnData-aware inspection. + +```bash +adata view data.h5ad # n_obs x n_var and the top-level keys +adata view data.h5ad --tree # a tree with each element's type +adata view data.h5ad --tree --depth 3 +adata view data.h5ad obsm/X_pca # detail for one entry +``` + +| Flag | Meaning | +|---|---| +| `--tree`, `-t` | Show a tree of all entries | +| `--depth N`, `-d` | Maximum recursion depth (with `--tree`) | + +`info` is a deprecated alias, removed in 1.0.0. + +## `ls` + +Format-agnostic listing. Makes no AnnData assumptions, so it also works on +`.loom` and plain `.h5` files. + +```bash +adata ls data.h5ad +adata ls data.h5ad --long # type, shape, dtype, encoding +adata ls data.h5ad uns --depth 2 # only below a path +adata ls data.h5ad -1 | grep spatial # bare paths, for piping +``` + +## `subset` + +Write a filtered copy. Select by name list, by expression, or both axes at +once. + +```bash +adata subset data.h5ad -o out.h5ad --obs barcodes.txt +adata subset data.h5ad -o out.h5ad --obs-query "cluster == Cortex_2" +adata subset data.h5ad -o out.h5ad -q "n_counts > 1000 and cluster in A,B" +adata subset data.h5ad -o out.h5ad --var-query "highly_variable == True" +adata subset data.h5ad --inplace --obs barcodes.txt +``` + +| Flag | Meaning | +|---|---| +| `--output`, `-o` | Output path. Required unless `--inplace` | +| `--inplace` | Replace the source (written to a temporary path first) | +| `--obs` / `--var` | File of names to keep, one per line | +| `--obs-query`, `-q` / `--var-query` | Keep rows matching an expression | +| `--chunk`, `-C` | Row chunk size for dense matrices | +| `--zarr-format` | Zarr version to write; defaults to the source's | + +`raw/` is carried over and matched against its **own** var axis, which usually +holds more genes than the main object. + +## `split` + +One store per distinct value of a column. + +```bash +adata split data.h5ad --by sample -o per_sample/ +adata split data.h5ad --by cell_type -o clusters/ --dry-run +adata split data.h5ad --by cluster -o out/ --min-size 50 +``` + +| Flag | Meaning | +|---|---| +| `--by`, `-b` | Column to split on | +| `--output-dir`, `-o` | Where to write the stores | +| `--axis` | `obs` (default) or `var` | +| `--min-size` | Skip groups smaller than this | +| `--dry-run` | Print the plan without writing | +| `--manifest / --no-manifest` | Write a CSV listing the outputs (default on) | + +Labels are sanitised for use as filenames; collisions get a numeric suffix. + +## `concat` + +Concatenate along the obs axis, streaming each input in turn. + +```bash +adata concat a.h5ad b.h5ad -o merged.h5ad +adata concat *.h5ad -o merged.h5ad --join outer --label sample +adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge same +``` + +| Flag | Meaning | +|---|---| +| `--join`, `-j` | `inner` (shared vars only) or `outer` (their union) | +| `--label` | Add an obs column recording each cell's source | +| `--keys` | Names for the inputs; defaults to their filenames | +| `--index-unique` | Delimiter for suffixing obs names with their key | +| `--merge` / `--uns-merge` | `same`, `unique`, `first`, `only`; default drops | +| `--fill-value` | Value for dense cells introduced by an outer join | + +obs columns keep their dtypes: categoricals union their category sets, nullable +columns keep their masks, and a column missing from one input is padded rather +than dropped. + +`obsp`, `varp` and `raw` are **not** carried over — a pairwise value has no +meaning between cells from different inputs. anndata's own `concat_on_disk` +refuses these too. + +### Merge strategies + +| Strategy | Keeps a value when | +|---|---| +| *(unset)* | never — the element is dropped | +| `same` | every input has it and they all agree | +| `unique` | exactly one distinct value among the inputs that have it | +| `first` | always; takes the earliest input that has it | +| `only` | exactly one input has it at all | + +## `create` + +Write a new, empty store for `import` to fill in. + +```bash +adata create out.h5ad --n-obs 5000 --n-var 2000 +adata create out.zarr --obs-names cells.txt --var-names genes.txt +``` + +Give each axis either a size (names are generated as `cell_0000`, `gene_0000`) +or a file of names. `--force` overwrites an existing store. + +## `export` + +| Subcommand | Produces | Accepts | +|---|---|---| +| `dataframe` | CSV | any dataframe path (`obs`, `var`, `raw/var`, …) | +| `array` | `.npy` | any dense array | +| `sparse` | `.mtx` | CSR/CSC groups | +| `dict` | JSON | any group or scalar | +| `image` | PNG/JPEG/TIFF | 2D or 3D arrays | + +```bash +adata export dataframe data.h5ad obs -o obs.csv +adata export dataframe data.h5ad obs --columns cluster,n_counts --head 100 +adata export sparse data.h5ad X -o matrix.mtx +adata export array data.h5ad obsm/X_umap > umap.npy +adata export dict data.h5ad uns -o metadata.json +``` + +Omit `--output` to write to stdout. Writing to a file streams in chunks; +stdout is not seekable, so `export array` holds the array in memory on that +path. + +## `import` + +Write data into a store at any path. All subcommands need `--output`/`-o` or +`--inplace`. + +```bash +adata import dataframe data.h5ad obs cells.csv --inplace -i cell_id +adata import array data.h5ad obsm/X_umap umap.npy --inplace +adata import sparse data.h5ad X counts.mtx --inplace +adata import dict data.h5ad uns/params params.json --inplace +adata import image data.h5ad uns/spatial/hires tissue.png --inplace +``` + +Dimensions are validated against the existing obs/var where the path implies +an axis. + +For `dataframe`: columns that parse as integers or floats become numeric +arrays; string columns with few distinct values become categoricals. Use +`--categorical col1,col2` to force specific columns, or +`--no-auto-categorical` to keep all strings as `string-array`. + +Giving `-o out.zarr` for an `.h5ad` source (or the reverse) converts the store +as it writes. + +## Query expressions + +Used by `subset --obs-query` and `--var-query`. + +| Operator | Example | +|---|---| +| `==`, `!=` | `cluster == Cortex_2` | +| `<`, `<=`, `>`, `>=` | `n_counts > 1000` | +| `in`, `not in` | `cluster in A,B,C` | +| `and`, `or`, `not` | `n_counts > 500 and not cluster == Fiber_tract` | +| parentheses | `(a == 1 or b == 2) and c > 3` | + +Values with spaces need quoting: `label == "cell type A"`. Equality and +membership compare the column's string form; the ordering operators parse it as +a number, and entries that will not parse simply do not match. + +Only the columns a query mentions are read, so filtering on one annotation does +not touch the rest of the frame. + +This is deliberately not SQL. For anything more involved — joins, aggregates, +window functions — export to CSV and use [duckdb](https://duckdb.org): + +```bash +adata export dataframe data.h5ad obs -o cells.csv +duckdb -noheader -list -c \ + "SELECT _index FROM 'cells.csv' WHERE cluster='Cortex_2' AND n_counts > 1000" \ + > barcodes.txt +adata subset data.h5ad -o cortex.h5ad --obs barcodes.txt +``` + +## Zarr format selection + +New Zarr stores follow the source store's version, so a v2 input is not +silently upgraded. `--zarr-format 2|3` on `create`, `subset`, `split` and +`concat` overrides that. Writing a Zarr store from an `.h5ad` source defaults +to v3. diff --git a/docs/ELEMENTS_h5ad.md b/docs/ELEMENTS_h5ad.md index 2cdd8db..62577fa 100644 --- a/docs/ELEMENTS_h5ad.md +++ b/docs/ELEMENTS_h5ad.md @@ -280,6 +280,42 @@ Group members: datasets for the buffers (often named like `nodeX-*`). > > This encoding is considered experimental in the anndata 0.9.x series and later. +## What `adata-cli` does with these elements + +This tool reads every layout listed above, including the legacy 0.7.x forms, +and always writes the current spec version shown in each section. + +| Element | Read | Written | +|---|---|---| +| `anndata` | yes | yes (0.1.0, stamped on every store it creates) | +| `raw` | yes | yes (0.1.0; subset against its own var axis) | +| `dict` | yes | yes (0.1.0, on every mapping group) | +| `dataframe` | 0.2.0 and legacy 0.1.0 | 0.2.0, with `column-order` | +| `array` | yes | yes (0.2.0) | +| `csr_matrix` / `csc_matrix` | yes | yes (0.1.0); both are streamed, never loaded whole | +| `categorical` | 0.2.0, plus both legacy layouts | 0.2.0, preserving `ordered` | +| `string-array` | yes | yes (0.2.0), variable-length UTF-8 | +| `nullable-integer` / `-boolean` / `-string-array` | yes | yes (0.1.0) | +| `numeric-scalar` | yes | yes (0.2.0) | +| `string` | yes | yes (0.2.0), as a 0-d dataset | +| `null` | yes | yes (0.1.0) | +| `awkward-array` | reported by `view` and `ls` | not written | + +### `null` (`encoding-version: 0.1.0`) + +Not in the upstream prose spec, but written by anndata 0.12+ for a `None` +value in `uns`. In HDF5 it is a dataset with a null dataspace (`h5py.Empty`); +in Zarr it is a 0-d boolean array. Both carry `encoding-type: null`. + +### Elements with no `encoding-type` + +Files written by anndata 0.7.x carry no encoding attributes at all. These are +classified structurally: a group with `codes` and `categories` is a +categorical, one with `values` and `mask` is a nullable array, one with +`_index` in its attributes is a dataframe, and anything else is a mapping. +Structural inference is only ever a fallback -- a declared `encoding-type` +always wins. + ## Sources - AnnData “on-disk format” prose docs (modern, ≥0.8): https://anndata.readthedocs.io/en/stable/fileformat-prose.html diff --git a/docs/ELEMENTS_zarr.md b/docs/ELEMENTS_zarr.md index f547024..46bae74 100644 --- a/docs/ELEMENTS_zarr.md +++ b/docs/ELEMENTS_zarr.md @@ -282,6 +282,69 @@ Group members: arrays for the buffers (often named like `nodeX-*`). > > This encoding is considered experimental in the anndata 0.9.x series and later. +## What `adata-cli` does with these elements + +This tool reads every layout listed above, including the legacy 0.7.x forms, +and always writes the current spec version shown in each section. + +| Element | Read | Written | +|---|---|---| +| `anndata` | yes | yes (0.1.0, stamped on every store it creates) | +| `raw` | yes | yes (0.1.0; subset against its own var axis) | +| `dict` | yes | yes (0.1.0, on every mapping group) | +| `dataframe` | 0.2.0 and legacy 0.1.0 | 0.2.0, with `column-order` | +| `array` | yes | yes (0.2.0) | +| `csr_matrix` / `csc_matrix` | yes | yes (0.1.0); both are streamed, never loaded whole | +| `categorical` | 0.2.0, plus both legacy layouts | 0.2.0, preserving `ordered` | +| `string-array` | yes | yes (0.2.0), variable-length UTF-8 | +| `nullable-integer` / `-boolean` / `-string-array` | yes | yes (0.1.0) | +| `numeric-scalar` | yes | yes (0.2.0) | +| `string` | yes | yes (0.2.0), as a 0-d dataset | +| `null` | yes | yes (0.1.0) | +| `awkward-array` | reported by `view` and `ls` | not written | + +### `null` (`encoding-version: 0.1.0`) + +Not in the upstream prose spec, but written by anndata 0.12+ for a `None` +value in `uns`. In HDF5 it is a dataset with a null dataspace (`h5py.Empty`); +in Zarr it is a 0-d boolean array. Both carry `encoding-type: null`. + +### Elements with no `encoding-type` + +Files written by anndata 0.7.x carry no encoding attributes at all. These are +classified structurally: a group with `codes` and `categories` is a +categorical, one with `values` and `mask` is a nullable array, one with +`_index` in its attributes is a dataframe, and anything else is a mapping. +Structural inference is only ever a fallback -- a declared `encoding-type` +always wins. + +### Zarr v2 versus v3 + +`zarr-python` 3 defaults to writing **v3**, and anndata 0.13 writes v3 by +default too. Both versions are readable here, and a derived store keeps the +source store's version rather than being silently upgraded; pass +`--zarr-format 2|3` to choose explicitly. + +The differences that matter when copying between stores: + +| | v2 | v3 | +|---|---|---| +| Group metadata | `.zgroup` / `.zattrs` | `zarr.json` | +| Array metadata | `.zarray` | `zarr.json` | +| Variable-length text | `VLenUTF8` in `filters` | `VariableLengthUTF8` data type | +| Compression | `compressor` (single) | `compressors` (a sequence) | +| Sharding | not available | `shards` | + +A v2 string array's `VLenUTF8` filter cannot be forwarded to a v3 array -- it +raises `Expected an ArrayArrayCodec`, because the v3 string data type encodes +variable length itself. This tool drops the filter and resolves the dtype for +the target rather than reusing the source's. + +Note also that `zarr-python` flags both `NullTerminatedBytes` (what `dtype="S"` +produces) and `FixedLengthUTF32` (what `1{print $4}' cells.csv | sort | uniq -c 192 Thalamus_2 ``` -To get all obs names in "Cortex_2", you can use `csvsql` from `csvkit`: +### Filtering directly + +For a filter this simple, `--obs-query` does the whole job without an +intermediate file: + +```bash +adata subset visium.h5ad --output cortex2.h5ad --obs-query "cluster == Cortex_2" +``` + +The expression language covers `==`, `!=`, `<`, `<=`, `>`, `>=`, `in`, +`not in`, `and`, `or`, `not` and parentheses, so most real filters fit: + +```bash +adata subset visium.h5ad -o big_cortex.h5ad \ + -q "cluster in Cortex_1,Cortex_2 and total_counts > 20000" +``` + +Only the columns the query mentions are read, so this does not touch the rest +of the frame. + +### Filtering with duckdb + +When a filter needs more than that — a join against another table, an +aggregate, a window function — export `obs` and let +[duckdb](https://duckdb.org) do the query, then feed the resulting names back +in: + ```bash -csvsql -d ',' -I --query "SELECT _index FROM cells WHERE cluster='Cortex_2'" cells.csv > barcodes.txt -sed -i '1d' barcodes.txt # remove header +duckdb -noheader -list -c \ + "SELECT _index FROM 'cells.csv' WHERE cluster='Cortex_2'" > barcodes.txt wc -l barcodes.txt # 257 barcodes.txt ``` -Now you can use this list to create a subset `.h5ad` file: +duckdb reads the CSV in place — no import step — and `-noheader -list` gives +one bare name per line, which is exactly the format `--obs` expects: + ```bash adata subset visium.h5ad --output cortex2.h5ad --obs barcodes.txt ``` +It is also a quicker way to do the cluster tally above: + +```bash +duckdb -c "SELECT cluster, count(*) FROM 'cells.csv' GROUP BY 1 ORDER BY 2 DESC" +``` + Check the result: ```bash adata view cortex2.h5ad diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..fdb09e6 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,11 @@ +# GitHub Pages configuration. Plain Jekyll with a built-in theme -- no +# generator to install, and the docs stay readable on github.com as markdown. +title: adata-cli +description: Streaming CLI for huge AnnData .h5ad and .zarr stores +theme: jekyll-theme-cayman +show_downloads: false + +# Site-relative links in the markdown files keep working on github.com too. +exclude: + - Gemfile + - Gemfile.lock diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..1c193dd --- /dev/null +++ b/docs/index.md @@ -0,0 +1,71 @@ +# adata-cli + +Explore and edit huge AnnData stores (`.h5ad` and `.zarr`) from the command +line, without loading them into memory and without a Python session. + +`adata-cli` implements the AnnData on-disk spec directly against `h5py` and +`zarr`. It has no dependency on `anndata`, `pandas` or `scipy`, which is what +lets it stream stores far larger than RAM — and what makes it useful in a +container or a pipeline step where installing the scientific stack would be +overkill. + +## Install + +```bash +pip install adata-cli +``` + +Or run it without installing anything: + +```bash +docker run --rm -it -v /path/to/data:/data \ + quay.io/cellgeni/adata-cli:latest adata view /data/your_file.h5ad +``` + +## Documentation + +- **[Get started](GET_STARTED.md)** — a hands-on walkthrough: inspect a store, + export metadata, filter it, write a subset. +- **[Command reference](COMMANDS.md)** — every command and flag. +- **[Element spec: HDF5](ELEMENTS_h5ad.md)** — how each AnnData element is laid + out in `.h5ad`, and what this tool does with it. +- **[Element spec: Zarr](ELEMENTS_zarr.md)** — the same for `.zarr`, including + the v2/v3 differences. + +## At a glance + +```bash +adata view data.h5ad # what's in it +adata ls data.h5ad --long # every path, with shapes and encodings +adata export dataframe data.h5ad obs # obs as CSV, on stdout + +adata subset data.h5ad -o cortex.h5ad --obs-query "cluster == Cortex_2" +adata split data.h5ad --by sample -o per_sample/ +adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample + +adata create new.h5ad --n-obs 5000 --n-var 2000 +adata import sparse new.h5ad X counts.mtx --inplace +``` + +## Format support + +Every AnnData on-disk layout is readable, from anndata 0.7.x through the +current spec; everything written is in the current spec. Both backends are +supported in both directions, so `adata subset in.h5ad -o out.zarr` converts as +it filters. + +| Element | Read | Write | +|---|---|---| +| `anndata`, `raw`, `dict` | yes | yes | +| `dataframe` (0.2.0 and legacy 0.1.0) | yes | 0.2.0 | +| `array` | yes | yes | +| `csr_matrix` / `csc_matrix` | yes | yes | +| `categorical` (incl. `ordered`, legacy layouts) | yes | yes | +| `string-array` | yes | yes | +| `nullable-integer` / `-boolean` / `-string-array` | yes | yes | +| `numeric-scalar`, `string`, `null` | yes | yes | +| `awkward-array` | shown by `view`/`ls` | no | + +## Source + +[github.com/cellgeni/adata-cli](https://github.com/cellgeni/adata-cli) diff --git a/pyproject.toml b/pyproject.toml index e90c4e6..7e7ff86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,11 @@ [project] name = "adata-cli" -version = "0.4.0.dev0" +version = "0.5.0" description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores" readme = "README.md" requires-python = ">=3.12" license = "MIT" +license-files = ["LICENSE"] authors = [ { name = "Aljes Binkevich", email = "ab76@sanger.ac.uk" }, ] @@ -14,18 +15,22 @@ maintainers = [ keywords = [ "anndata", "h5ad", + "zarr", "bioinformatics", "single-cell", + "cli", ] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Environment :: Console", "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Bio-Informatics", + "Typing :: Typed", ] dependencies = [ @@ -38,10 +43,11 @@ dependencies = [ ] [project.urls] -Homepage = "https://github.com/cellgeni/h5ad-cli" -Repository = "https://github.com/cellgeni/h5ad-cli" -Documentation = "README.md" -Issues = "https://github.com/cellgeni/h5ad-cli/issues" +Homepage = "https://github.com/cellgeni/adata-cli" +Repository = "https://github.com/cellgeni/adata-cli" +Documentation = "https://cellgeni.github.io/adata-cli/" +Issues = "https://github.com/cellgeni/adata-cli/issues" +Changelog = "https://github.com/cellgeni/adata-cli/blob/main/CHANGELOG.md" [project.optional-dependencies] dev = [ @@ -59,6 +65,7 @@ build-backend = "uv_build" [tool.uv.build-backend] module-name = "adata" + [project.scripts] adata = "adata.cli:main" # Deprecated alias, removed in 1.0.0 diff --git a/src/adata/__init__.py b/src/adata/__init__.py index e69de29..770b20d 100644 --- a/src/adata/__init__.py +++ b/src/adata/__init__.py @@ -0,0 +1,10 @@ +"""adata-cli: streaming CLI for large AnnData .h5ad and .zarr stores.""" + +from importlib.metadata import PackageNotFoundError, version as _version + +try: + __version__ = _version("adata-cli") +except PackageNotFoundError: # pragma: no cover - running from a source tree + __version__ = "0.0.0+unknown" + +__all__ = ["__version__"] diff --git a/src/adata/cli.py b/src/adata/cli.py index 9da575f..d3b3118 100644 --- a/src/adata/cli.py +++ b/src/adata/cli.py @@ -24,7 +24,7 @@ app = typer.Typer( help="Streaming CLI for huge AnnData .h5ad and .zarr stores " - "(view, ls, subset, export, import)." + "(view, ls, subset, split, concat, export, import)." ) # Use stderr for status/progress to keep stdout clean for data output # force_terminal=True ensures Rich output is visible even in non-TTY environments @@ -32,6 +32,15 @@ # Results go to stdout so they can be piped; status and errors stay on stderr. out_console = Console() + +def _version_callback(value: bool) -> None: + if value: + from adata import __version__ + + out_console.print(__version__, highlight=False) + raise typer.Exit() + + # Create sub-apps for export and import export_app = typer.Typer(help="Export objects from an AnnData store.") import_app = typer.Typer(help="Import objects into an AnnData store.") @@ -39,8 +48,22 @@ app.add_typer(import_app, name="import") +@app.callback() +def _root( + version: bool = typer.Option( + False, + "--version", + "-V", + help="Show the installed version and exit.", + callback=_version_callback, + is_eager=True, + ), +) -> None: + """Streaming CLI for huge AnnData .h5ad and .zarr stores.""" + + # ============================================================================ -# INFO command +# VIEW command # ============================================================================ @app.command("view") def view( diff --git a/src/adata/py.typed b/src/adata/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/uv.lock b/uv.lock index 0e90f12..256d4fb 100644 --- a/uv.lock +++ b/uv.lock @@ -12,7 +12,7 @@ resolution-markers = [ [[package]] name = "adata-cli" -version = "0.4.0.dev0" +version = "0.5.0" source = { editable = "." } dependencies = [ { name = "h5py" }, From 7a0eb333fc25ab2890e41627d261809031623f1a Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 12:30:34 +0100 Subject: [PATCH 13/23] Grant pull-requests: write so the test-results action can comment The action posts its summary as a PR comment, which needs pull-requests: write. Without it the step 403s and fails the job even when every test passed, which is what happened on #6. Co-Authored-By: Claude Opus 5 --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f12a9d1..220f9a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,8 @@ jobs: permissions: contents: read - checks: write # needed for EnricoMi/publish-unit-test-result-action on PRs + checks: write # publish-unit-test-result-action writes a check run + pull-requests: write # ...and comments the summary on the PR strategy: fail-fast: false From 085515ec3b5fa16edfe466639a25d57d9091f511 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 12:32:15 +0100 Subject: [PATCH 14/23] Run tests on every pull request, not only those into main/dev A PR stacked on another feature branch matched neither branch filter, so it reported no checks at all. Co-Authored-By: Claude Opus 5 --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 220f9a3..ee0d2a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,8 +3,9 @@ name: Tests on: push: branches: [main, dev] + # Every pull request, not only those targeting main/dev -- a stacked PR + # based on another feature branch would otherwise run no checks at all. pull_request: - branches: [main, dev] concurrency: group: tests-${{ github.workflow }}-${{ github.ref }} From 70dc4d87f6f6197233a17ba7a9b277bf5a269627 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 12:41:31 +0100 Subject: [PATCH 15/23] Fix six correctness issues found in review of #7 CSC matrices were silently dropped by concat. `kinds == {"csc_matrix"}` matched no branch, so the code warned -- misreporting agreeing inputs as disagreeing -- and returned successfully, writing an output with no X at all. CSC is now concatenated as CSC: for each target column, each input's row indices are appended with that input's row offset added, which stays sorted without a re-sort. Encodings are also checked before the output store is created, so a genuine mismatch fails loudly instead of leaving a partial store. Default concat keys came from filename stems, so inputs in different directories sharing a name produced duplicate keys. --label writes those as categorical categories, which must be unique, so the command reported success while producing an output pandas could not read. Duplicates are now rejected, for explicit --keys too. `import image` replaced its destination unconditionally, so `import image data.h5ad obs img.png` deleted the obs dataframe. Images are now refused for paths that must hold a dataframe, and validated against the axis for everything else. `raw/var` was not recognised by validate_dimensions, so it could be replaced at any length while raw/X kept its width. raw/var, raw/X and raw/varm/* are now validated against raw's own var axis, taken from raw/X. Under --uns-merge same/unique, non-dict groups were compared by their child key names alone, so two dataframes with the same columns but different values looked equal and the first input's was kept. Comparison now recurses into contents and attributes, with a size cap above which an element is treated as equal to nothing. Dict groups already recursed, which is why this only showed up for dataframes, sparse matrices and categoricals. A nullable var column surviving a merge was rendered as text, turning missing values into empty strings and losing the numeric dtype. It now keeps its values, mask, encoding and na-value. Each fix has a regression test that fails without it. Co-Authored-By: Claude Opus 5 --- src/adata/core/concat.py | 260 +++++++++++++++++++++++++++++++--- src/adata/formats/image.py | 12 ++ src/adata/formats/validate.py | 83 +++++++++++ tests/test_commands_phase2.py | 243 +++++++++++++++++++++++++++++++ 4 files changed, 581 insertions(+), 17 deletions(-) diff --git a/src/adata/core/concat.py b/src/adata/core/concat.py index 4b6cea9..4982916 100644 --- a/src/adata/core/concat.py +++ b/src/adata/core/concat.py @@ -86,15 +86,43 @@ def _apply_index_unique( def _resolve_keys( files: Sequence[Path], keys: Optional[Sequence[str]] ) -> List[str]: + """Name each input, defaulting to its filename stem. + + Keys must be distinct: `--label` writes them as categorical categories, + which have to be unique, and `--index-unique` uses them to disambiguate obs + names. Duplicates would otherwise produce an output that reports success + but cannot be read back. + """ if keys is None: - return [f.stem if f.suffix else f.name for f in files] + resolved = [f.stem if f.suffix else f.name for f in files] + duplicates = _duplicates(resolved) + if duplicates: + raise ValueError( + f"Inputs in different directories share the filename(s) " + f"{', '.join(duplicates)}, so the default keys are not unique. " + "Pass --keys to name them explicitly." + ) + return resolved + if len(keys) != len(files): raise ValueError( f"--keys has {len(keys)} entries but {len(files)} inputs were given." ) + duplicates = _duplicates(keys) + if duplicates: + raise ValueError( + f"--keys must be unique; repeated: {', '.join(duplicates)}." + ) return list(keys) +def _duplicates(values: Sequence[str]) -> List[str]: + seen: Dict[str, int] = {} + for value in values: + seen[value] = seen.get(value, 0) + 1 + return sorted(v for v, n in seen.items() if n > 1) + + # --------------------------------------------------------------------------- # merge strategies @@ -147,6 +175,8 @@ def __repr__(self) -> str: # pragma: no cover - debugging aid def _equal(a: Any, b: Any) -> bool: + if isinstance(a, _Incomparable) or isinstance(b, _Incomparable): + return False try: if isinstance(a, np.ndarray) or isinstance(b, np.ndarray): return np.array_equal(np.asarray(a), np.asarray(b)) @@ -436,18 +466,118 @@ def _concat_dense( row_offset += n_src_rows +def _inverse_column_map(col_map: np.ndarray, n_cols: int) -> np.ndarray: + """Invert a source->target column map into target->source, -1 where absent.""" + inverse = np.full(n_cols, -1, dtype=np.int64) + present = col_map >= 0 + inverse[col_map[present]] = np.nonzero(present)[0] + return inverse + + +def _concat_csc( + dst_parent: Any, + name: str, + sources: List[Any], + col_maps: List[np.ndarray], + row_counts: List[int], + n_rows: int, + n_cols: int, +) -> None: + """Concatenate CSC matrices row-wise, keeping the CSC encoding. + + A CSC matrix is stored by column, so concatenating along obs means, for + each target column, appending each input's row indices in turn with that + input's row offset added. Because every input's column is already sorted + and the offsets increase, the result is sorted without a re-sort. + """ + from adata.core.subset import _append, _growable + from adata.elements.write import set_shape_attr + + group = dst_parent.create_group(name) + spec.set_encoding(group, spec.CSC_MATRIX) + set_shape_attr(group, (n_rows, n_cols)) + + dtype = np.result_type(*[s["data"].dtype for s in sources]) + out_data = _growable(group, "data", dtype) + out_indices = _growable(group, "indices", np.int64) + indptr = [0] + nnz = 0 + + inverses = [_inverse_column_map(cm, n_cols) for cm in col_maps] + indptrs = [np.asarray(s["indptr"][...], dtype=np.int64) for s in sources] + row_offsets = np.cumsum([0] + list(row_counts[:-1])) + + for target_col in range(n_cols): + rows: List[np.ndarray] = [] + values: List[np.ndarray] = [] + for source, src_indptr, inverse, offset in zip( + sources, indptrs, inverses, row_offsets + ): + src_col = int(inverse[target_col]) + if src_col < 0: + continue + lo, hi = int(src_indptr[src_col]), int(src_indptr[src_col + 1]) + if hi <= lo: + continue + rows.append(np.asarray(source["indices"][lo:hi], dtype=np.int64) + offset) + values.append(np.asarray(source["data"][lo:hi])) + + if rows: + _append(out_indices, np.concatenate(rows)) + _append(out_data, np.concatenate(values).astype(dtype)) + nnz += int(sum(len(r) for r in rows)) + indptr.append(nnz) + + create_dataset(group, "indptr", data=np.asarray(indptr, dtype=np.int64)) + + +def check_matrix_encodings(roots: List[Any], console: Console) -> None: + """Fail before writing anything if a matrix cannot be concatenated. + + Checked up front rather than mid-write: discovering this half way through + would leave a partial store behind, and skipping the matrix would produce + an output silently missing X. + """ + def _check(label: str, sources: List[Any]) -> None: + kinds = {_matrix_kind(s) for s in sources} + if kinds in ({spec.CSR_MATRIX}, {spec.CSC_MATRIX}, {"dense"}): + return + raise ValueError( + f"Cannot concatenate {label!r}: inputs use " + f"{', '.join(sorted(kinds))}. Every input must use the same " + "encoding -- convert them to match first." + ) + + if all("X" in r for r in roots): + _check("X", [r["X"] for r in roots]) + elif any("X" in r for r in roots): + console.print("[yellow]Skipping X: not present in every input[/]") + + names = _index_union( + [list(r["layers"].keys()) if "layers" in r else [] for r in roots] + ) + for name in names: + if all("layers" in r and name in r["layers"] for r in roots): + _check(f"layers/{name}", [r["layers"][name] for r in roots]) + + def _concat_matrix( dst_parent: Any, name: str, sources: List[Any], col_maps: List[np.ndarray], + row_counts: List[int], n_rows: int, n_cols: int, chunk_rows: int, fill_value: float, console: Console, ) -> bool: - """Concatenate X or one layer across inputs. Returns whether it was written.""" + """Concatenate X or one layer across inputs. Returns whether it was written. + + Encodings are validated by :func:`check_matrix_encodings` before the output + store exists, so anything reaching here is concatenable. + """ kinds = {_matrix_kind(s) for s in sources} if kinds == {spec.CSR_MATRIX}: @@ -456,6 +586,12 @@ def _concat_matrix( ) return True + if kinds == {spec.CSC_MATRIX}: + _concat_csc( + dst_parent, name, sources, col_maps, row_counts, n_rows, n_cols + ) + return True + if kinds == {"dense"}: _concat_dense( dst_parent, @@ -469,11 +605,9 @@ def _concat_matrix( ) return True - console.print( - f"[yellow]Skipping {name!r}: inputs disagree on encoding " - f"({', '.join(sorted(kinds))}). Convert them to match first.[/]" + raise ValueError( + f"Cannot concatenate {name!r}: inputs use {', '.join(sorted(kinds))}." ) - return False def _concat_obsm( @@ -559,14 +693,68 @@ def _merge_group( copy_tree(source, target, key) -def _readable_value(obj: Any) -> Any: - """A comparable snapshot of a small element, for merge strategies.""" +#: Elements larger than this are not compared value-by-value. +MAX_COMPARABLE_ELEMENTS = 1_000_000 + + +class _Incomparable: + """Stands for a value too large to compare, and equal to nothing.""" + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return "" + + +def _readable_value(obj: Any, budget: Optional[List[int]] = None) -> Any: + """A comparable snapshot of an element, for the merge strategies. + + Recurses into groups and reads their datasets, including attributes. Using + only child key names -- as an earlier version did -- would make two + dataframes with the same columns but different values compare equal, so + `same` and `unique` would keep conflicting metadata. + + Very large elements are reported as incomparable, which equals nothing and + so is never kept by `same` or `unique`. + """ + if budget is None: + budget = [MAX_COMPARABLE_ELEMENTS] + try: + attrs = tuple( + sorted( + (str(k), _hashable(spec.decode_attr(v))) + for k, v in obj.attrs.items() + if k not in ("encoding-version",) + ) + ) + if is_dataset(obj): - return np.asarray(obj[...]) - return tuple(sorted(obj.keys())) + size = int(np.prod(obj.shape)) if obj.shape else 1 + budget[0] -= size + if budget[0] < 0: + return _Incomparable() + return ("dataset", attrs, _hashable(np.asarray(obj[...]))) + + children = [] + for key in sorted(obj.keys()): + value = _readable_value(obj[key], budget) + if isinstance(value, _Incomparable): + return value + children.append((str(key), value)) + return ("group", attrs, tuple(children)) except Exception: - return _MISSING + return _Incomparable() + + +def _hashable(value: Any) -> Any: + """Reduce a value to something `_equal` can compare reliably.""" + if isinstance(value, np.ndarray): + return (value.shape, value.dtype.kind, value.tobytes() + if value.dtype.kind not in ("O", "T") else tuple(map(str, value.reshape(-1).tolist()))) + if isinstance(value, np.generic): + return value.item() + if isinstance(value, (list, tuple)): + return tuple(_hashable(v) for v in value) + return value def concat_on_disk( @@ -644,6 +832,10 @@ def concat_on_disk( "Pass --index-unique to disambiguate them.[/]" ) + # Validated before the output exists, so a mismatch cannot leave a + # half-written store behind. + check_matrix_encodings(roots, console) + with open_store(output, "w", zarr_format=zarr_format) as dst_store: dst = dst_store.root _write_obs( @@ -653,7 +845,15 @@ def concat_on_disk( ensure_anndata_skeleton(dst) _write_matrices( - dst, roots, col_maps, n_obs, n_var, chunk_rows, fill_value, console + dst, + roots, + col_maps, + obs_counts, + n_obs, + n_var, + chunk_rows, + fill_value, + console, ) _write_obsm(dst, roots, n_obs, chunk_rows, console) @@ -771,25 +971,49 @@ def _write_var( def _write_var_column( parent: Any, name: str, column: Any, take: np.ndarray ) -> None: - """Write one var column, reordered onto the target var index.""" + """Write one var column, reordered onto the target var index. + + Each encoding is rewritten as itself. Rendering everything as text -- as an + earlier version did for nullable columns -- turned missing values into + empty strings and lost the numeric and boolean dtypes. + """ kind = _column_kind(column) + if kind == "categorical": categories = [str(c) for c in read_categories(column)] codes = np.asarray(column["codes"][...], dtype=np.int64)[take] write_categorical( parent, name, codes, categories, ordered=is_ordered(column) ) - elif kind == "numeric": + return + + if kind == "numeric": write_dense(parent, name, np.asarray(column[...])[take]) - else: - values = read_str_all(column) - write_string_array(parent, name, [values[i] for i in take]) + return + + if kind == "masked": + enc = spec.encoding_type(column) or spec.NULLABLE_STRING_ARRAY + mask = np.asarray(column["mask"][...], dtype=bool)[take] + raw_values = np.asarray(column["values"][...]) + if enc == spec.NULLABLE_STRING_ARRAY: + from adata.elements.read import decode_str_array + + values = decode_str_array(raw_values)[take].tolist() + else: + values = raw_values[take] + na_value = spec.decode_attr(column.attrs.get("na-value", None)) + write_masked(parent, name, values, mask, enc, na_value=na_value) + return + + values = read_str_all(column) + write_string_array(parent, name, [values[i] for i in take]) def _write_matrices( dst: Any, roots: List[Any], col_maps: List[np.ndarray], + row_counts: List[int], n_obs: int, n_var: int, chunk_rows: int, @@ -803,6 +1027,7 @@ def _write_matrices( "X", [r["X"] for r in roots], col_maps, + row_counts, n_obs, n_var, chunk_rows, @@ -828,6 +1053,7 @@ def _write_matrices( name, [r["layers"][name] for r in roots], col_maps, + row_counts, n_obs, n_var, chunk_rows, diff --git a/src/adata/formats/image.py b/src/adata/formats/image.py index 779f2bc..c897eab 100644 --- a/src/adata/formats/image.py +++ b/src/adata/formats/image.py @@ -8,6 +8,7 @@ from rich.console import Console from adata.elements.write import write_dense, write_mapping +from adata.formats.validate import DATAFRAME_PATHS, validate_dimensions from adata.formats.common import _resolve from adata.storage import is_dataset from adata.util.path import norm_path @@ -61,6 +62,17 @@ def import_image(root: Any, obj: str, input_file: Path, console: Console) -> Non if arr.ndim not in (2, 3): raise ValueError(f"Expected a 2D or 3D image; got shape {arr.shape}.") + if obj in DATAFRAME_PATHS: + raise ValueError( + f"'{obj}' must hold a dataframe; writing an image there would " + "corrupt the store. Images belong somewhere unstructured, such as " + "'uns/spatial/hires'." + ) + + # An image's height is not an axis, so a path that implies one is almost + # certainly a mistake -- but check it rather than assume. + validate_dimensions(root, obj, arr.shape, console) + parts = obj.split("/") parent = root for part in parts[:-1]: diff --git a/src/adata/formats/validate.py b/src/adata/formats/validate.py index ce027d2..2d30b49 100644 --- a/src/adata/formats/validate.py +++ b/src/adata/formats/validate.py @@ -12,6 +12,10 @@ VAR_AXIS_PREFIXES = ("var", "varm/", "varp/") MATRIX_PREFIXES = ("X", "layers/") +#: Paths that must hold a dataframe, so writing an array there would corrupt +#: the store rather than merely mis-size it. +DATAFRAME_PATHS = frozenset({"obs", "var", "raw/var"}) + def _get_axis_length(root: Any, axis: str) -> Optional[int]: try: @@ -20,6 +24,81 @@ def _get_axis_length(root: Any, axis: str) -> Optional[int]: return None +def _raw_var_length(root: Any) -> Optional[int]: + """Number of variables in `raw`, which is its own axis. + + Taken from `raw/X`'s declared width where possible: that is the invariant + a replacement `raw/var` has to keep, and it usually differs from the main + object's var count. + """ + if "raw" not in root: + return None + raw = root["raw"] + + if "X" in raw: + shape = raw["X"].attrs.get("shape", None) + if shape is not None and len(shape) >= 2: + return int(shape[1]) + x_shape = getattr(raw["X"], "shape", None) + if x_shape is not None and len(x_shape) >= 2: + return int(x_shape[1]) + + if "var" in raw: + try: + return axis_len(raw, "var") + except Exception: + return None + return None + + +def _validate_raw( + root: Any, obj_path: str, data_shape: Tuple[int, ...], console: Console +) -> bool: + """Validate a path under `raw/`. Returns whether the path was recognised.""" + if obj_path == "raw" or not obj_path.startswith("raw/"): + return False + + n_raw_var = _raw_var_length(root) + n_obs = _get_axis_length(root, "obs") + rest = obj_path[len("raw/"):] + + if rest == "var": + if n_raw_var is not None and data_shape[0] != n_raw_var: + raise ValueError( + f"Row count mismatch: input has {data_shape[0]} rows, but raw " + f"has {n_raw_var} variables. Replacing raw/var with a " + "different length would leave raw/X inconsistent." + ) + return True + + if rest == "X": + if len(data_shape) < 2: + raise ValueError( + f"raw/X requires 2D data, got {len(data_shape)}D." + ) + if n_obs is not None and data_shape[0] != n_obs: + raise ValueError( + f"First dimension mismatch: input has {data_shape[0]} rows, " + f"but obs has {n_obs} cells." + ) + if n_raw_var is not None and data_shape[1] != n_raw_var: + raise ValueError( + f"Second dimension mismatch: input has {data_shape[1]} columns, " + f"but raw has {n_raw_var} variables." + ) + return True + + if rest.startswith("varm/"): + if n_raw_var is not None and data_shape[0] != n_raw_var: + raise ValueError( + f"First dimension mismatch: input has {data_shape[0]} rows, " + f"but raw has {n_raw_var} variables." + ) + return True + + return False + + def validate_dimensions( root: Any, obj_path: str, @@ -27,6 +106,10 @@ def validate_dimensions( console: Console, ) -> None: obj_path = norm_path(obj_path) + + if _validate_raw(root, obj_path, data_shape, console): + return + n_obs = _get_axis_length(root, "obs") n_var = _get_axis_length(root, "var") diff --git a/tests/test_commands_phase2.py b/tests/test_commands_phase2.py index d67bd62..d7b8ec8 100644 --- a/tests/test_commands_phase2.py +++ b/tests/test_commands_phase2.py @@ -450,3 +450,246 @@ def test_concat_rejects_an_unknown_merge_strategy(tmp_path): ) assert result.exit_code == 1 assert "must be one of" in _out(result) + + +# --------------------------------------------------------------------------- +# regressions from review of #7 + + +@pytest.mark.parametrize("layout", ["csr", "csc"]) +def test_concat_preserves_sparse_layout(tmp_path, layout): + """CSC inputs were silently dropped, leaving an output with no X.""" + rng = np.random.default_rng(4) + mats = [ + sparse.csr_matrix(rng.poisson(1.0, (2, 3)).astype("float32")) + for _ in range(2) + ] + if layout == "csc": + mats = [m.tocsc() for m in mats] + + paths = [] + for i, (mat, cells) in enumerate(zip(mats, (["c1", "c2"], ["c3", "c4"]))): + obj = ad.AnnData( + X=mat, + obs=pd.DataFrame(index=cells), + var=pd.DataFrame(index=["g1", "g2", "g3"]), + ) + obj.layers["counts"] = mat.copy() + path = tmp_path / f"{i}.h5ad" + obj.write_h5ad(path) + paths.append(str(path)) + + out = tmp_path / "m.h5ad" + result = runner.invoke(app, ["concat", *paths, "-o", str(out)]) + assert result.exit_code == 0, _out(result) + + got = ad.read_h5ad(out) + assert got.X is not None, "X must not be silently omitted" + assert type(got.X).__name__ == f"{layout}_matrix" + assert "counts" in got.layers + + expected = ad.concat([ad.read_h5ad(p) for p in paths]) + assert np.array_equal( + np.asarray(got.X.todense()), np.asarray(expected.X.todense()) + ) + + +def test_concat_rejects_mixed_sparse_encodings_before_writing(tmp_path): + """A mismatch must fail loudly, and leave no half-written store behind.""" + rng = np.random.default_rng(5) + base = rng.poisson(1.0, (2, 3)).astype("float32") + for i, (mat, cells) in enumerate( + zip( + [sparse.csr_matrix(base), sparse.csc_matrix(base)], + (["c1", "c2"], ["c3", "c4"]), + ) + ): + ad.AnnData( + X=mat, + obs=pd.DataFrame(index=cells), + var=pd.DataFrame(index=["g1", "g2", "g3"]), + ).write_h5ad(tmp_path / f"{i}.h5ad") + + out = tmp_path / "m.h5ad" + result = runner.invoke( + app, + ["concat", str(tmp_path / "0.h5ad"), str(tmp_path / "1.h5ad"), + "-o", str(out)], + ) + assert result.exit_code == 1 + assert "same encoding" in _out(result) + assert not out.exists(), "a rejected concat must not leave a partial store" + + +def test_concat_rejects_duplicate_default_keys(tmp_path): + """Same filename in different directories made --label unreadable.""" + for sub, cells in (("run1", ["c1", "c2"]), ("run2", ["c3", "c4"])): + (tmp_path / sub).mkdir() + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=cells), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(tmp_path / sub / "sample.h5ad") + + args = [ + "concat", + str(tmp_path / "run1" / "sample.h5ad"), + str(tmp_path / "run2" / "sample.h5ad"), + "-o", str(tmp_path / "m.h5ad"), + ] + result = runner.invoke(app, [*args, "--label", "origin"]) + assert result.exit_code == 1 + assert "--keys" in _out(result) + + # Naming them explicitly resolves it. + result = runner.invoke( + app, [*args, "--label", "origin", "--keys", "run1,run2"] + ) + assert result.exit_code == 0, _out(result) + assert list(ad.read_h5ad(tmp_path / "m.h5ad").obs["origin"]) == [ + "run1", "run1", "run2", "run2" + ] + + +def test_concat_rejects_duplicate_explicit_keys(tmp_path): + a = _make(tmp_path / "a.h5ad", ["c1"], ["g1"], batch="A") + b = _make(tmp_path / "b.h5ad", ["c2"], ["g1"], batch="B") + result = runner.invoke( + app, + ["concat", str(a), str(b), "-o", str(tmp_path / "m.h5ad"), + "--keys", "same,same"], + ) + assert result.exit_code == 1 + assert "unique" in _out(result) + + +def test_concat_uns_merge_compares_values_not_layout(tmp_path): + """Non-dict uns groups were compared by child key names alone. + + A dataframe is the case that matters: two with identical columns but + different values looked equal, so `same` copied the first input's and kept + metadata the strategy was meant to reject. Dict groups already recursed, + so they never showed the bug. + """ + for name, scores in (("a", [1.0, 2.0]), ("b", [9.0, 9.0])): + obj = ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=["c1", "c2"] if name == "a" else ["c3", "c4"]), + var=pd.DataFrame(index=["g1", "g2"]), + ) + obj.uns["stats"] = pd.DataFrame( + {"score": scores}, index=["g1", "g2"] + ) + obj.uns["constant"] = pd.DataFrame( + {"score": [7.0, 7.0]}, index=["g1", "g2"] + ) + obj.write_h5ad(tmp_path / f"{name}.h5ad") + + out = tmp_path / "m.h5ad" + result = runner.invoke( + app, + ["concat", str(tmp_path / "a.h5ad"), str(tmp_path / "b.h5ad"), + "-o", str(out), "--uns-merge", "same"], + ) + assert result.exit_code == 0, _out(result) + + uns = ad.read_h5ad(out).uns + assert "stats" not in uns, "differing dataframes must not survive 'same'" + assert "constant" in uns, "identical dataframes should survive" + + +def test_concat_keeps_nullable_var_columns_nullable(tmp_path): + """A merged nullable var column kept its dtype instead of becoming text.""" + for name, cells in (("a", ["c1", "c2"]), ("b", ["c3", "c4"])): + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=cells), + var=pd.DataFrame( + {"nullable": pd.array([1, None], dtype="Int32")}, + index=["g1", "g2"], + ), + ).write_h5ad(tmp_path / f"{name}.h5ad") + + out = tmp_path / "m.h5ad" + result = runner.invoke( + app, + ["concat", str(tmp_path / "a.h5ad"), str(tmp_path / "b.h5ad"), + "-o", str(out), "--merge", "same"], + ) + assert result.exit_code == 0, _out(result) + + var = ad.read_h5ad(out).var + assert str(var["nullable"].dtype) == "Int32" + assert var["nullable"].isna().tolist() == [False, True] + + +def test_import_image_refuses_to_clobber_a_dataframe(tmp_path, sample): + """`import image ... obs` deleted the obs dataframe outright.""" + from PIL import Image + + png = tmp_path / "tissue.png" + Image.fromarray(np.zeros((8, 8, 3), dtype="uint8")).save(png) + + result = runner.invoke( + app, ["import", "image", str(sample), "obs", str(png), "--inplace"] + ) + assert result.exit_code == 1 + assert "must hold a dataframe" in _out(result) + + # The store is untouched. + assert ad.read_h5ad(sample).shape == (4, 2) + + +def test_import_image_validates_axis_bound_paths(tmp_path, sample): + from PIL import Image + + png = tmp_path / "tissue.png" + Image.fromarray(np.zeros((8, 8, 3), dtype="uint8")).save(png) + + # 8 rows against 4 obs. + result = runner.invoke( + app, ["import", "image", str(sample), "obsm/bad", str(png), "--inplace"] + ) + assert result.exit_code == 1 + assert "mismatch" in _out(result) + + # Unstructured destinations are fine. + result = runner.invoke( + app, + ["import", "image", str(sample), "uns/spatial/hires", str(png), + "--inplace"], + ) + assert result.exit_code == 0, _out(result) + assert ad.read_h5ad(sample).uns["spatial"]["hires"].shape == (8, 8, 3) + + +def test_import_dataframe_validates_raw_var_against_raw(tmp_path): + """raw/var was replaceable at any length, desynchronising it from raw/X.""" + obj = ad.AnnData( + X=np.ones((2, 4), dtype="float32"), + obs=pd.DataFrame(index=["c1", "c2"]), + var=pd.DataFrame(index=["g1", "g2", "g3", "g4"]), + ) + obj.raw = obj + src = tmp_path / "withraw.h5ad" + obj.write_h5ad(src) + + wrong = tmp_path / "two.csv" + wrong.write_text("_index,x\ng1,1\ng2,2\n") + result = runner.invoke( + app, + ["import", "dataframe", str(src), "raw/var", str(wrong), + "--inplace", "-i", "_index"], + ) + assert result.exit_code == 1 + assert "raw has 4 variables" in _out(result) + + right = tmp_path / "four.csv" + right.write_text("_index,x\ng1,1\ng2,2\ng3,3\ng4,4\n") + result = runner.invoke( + app, + ["import", "dataframe", str(src), "raw/var", str(right), + "--inplace", "-i", "_index"], + ) + assert result.exit_code == 0, _out(result) + assert list(ad.read_h5ad(src).raw.var["x"]) == [1, 2, 3, 4] From 1041c9d26eac769f87100ec9c50b4359c1139709 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 12:49:32 +0100 Subject: [PATCH 16/23] Fix JSON round-trip and Zarr consolidated-metadata bugs from review of #6 A nested string list such as [["a","b"],["c","d"]] was flattened into a length-4 vector on import, because the string branch of _write_json_list reshaped before writing. The shaped array is now passed through, and write_string_array no longer list()s an ndarray, which would flatten it again. Exporting a `null` element emitted its storage placeholder rather than None: the string "Empty(dtype=...)" on HDF5, and false on Zarr. _dataset_to_jsonable now recognises the encoding, so JSON payloads containing null round-trip exactly. Chasing those two through Zarr surfaced a larger one. anndata writes a consolidated metadata index at the root of a .zarr store. New members do land on disk, but every reader honouring that index -- anndata included -- keeps reading the stale snapshot, so an `--inplace` import into an anndata-written store reported success and then appeared to do nothing. Writable stores are now opened with use_consolidated=False and the index is rewritten on close. Only stores written by anndata were affected, which is why the existing Zarr tests, which build their own stores, never caught it. That in turn exposed subset_raw_group assuming `raw` is a group. anndata writes a null-encoded placeholder array there when an object has no raw, so on Zarr the subset crashed with "'Array' object has no attribute 'keys'". A non-group `raw` is now copied verbatim. Each fix has a regression test that fails without it. Co-Authored-By: Claude Opus 5 --- src/adata/core/subset.py | 10 ++- src/adata/elements/write.py | 13 +++- src/adata/formats/json_data.py | 11 ++- src/adata/storage/__init__.py | 26 ++++++- tests/test_anndata_roundtrip.py | 124 ++++++++++++++++++++++++++++++++ 5 files changed, 177 insertions(+), 7 deletions(-) diff --git a/src/adata/core/subset.py b/src/adata/core/subset.py index 1dba540..f58b92f 100644 --- a/src/adata/core/subset.py +++ b/src/adata/core/subset.py @@ -562,10 +562,16 @@ def subset_h5ad( tasks.extend([f"varp:{k}" for k in src["varp"].keys()]) if "uns" in src: tasks.append("uns") - if "raw" in src: + # anndata writes a placeholder `raw` even when there is none, and + # on Zarr that placeholder is an array rather than a group. + if "raw" in src and is_group(src["raw"]): tasks.append("raw") + elif "raw" in src: + tasks.append("copy:raw") - passthrough = [k for k in src.keys() if k not in HANDLED_KEYS] + passthrough = [ + k for k in src.keys() if k not in HANDLED_KEYS + ] if passthrough: console.print( "[yellow]Copying unrecognised top-level " diff --git a/src/adata/elements/write.py b/src/adata/elements/write.py index 4098967..46b3b29 100644 --- a/src/adata/elements/write.py +++ b/src/adata/elements/write.py @@ -39,10 +39,19 @@ def write_mapping(parent: Any, name: str, replace: bool = False) -> Any: def write_string_array( parent: Any, name: str, values: Iterable[Any], replace: bool = False ) -> Any: - """Write a `string-array`: variable-length UTF-8 on either backend.""" + """Write a `string-array`: variable-length UTF-8 on either backend. + + Multi-dimensional input keeps its shape; `list()`-ing it first would + silently flatten a 2-D array of labels into a vector. + """ if replace: _replace(parent, name) - ds = create_dataset(parent, name, data=np.asarray(list(values), dtype=object)) + data = ( + values + if isinstance(values, np.ndarray) + else np.asarray(list(values), dtype=object) + ) + ds = create_dataset(parent, name, data=data) spec.set_encoding(ds, spec.STRING_ARRAY) return ds diff --git a/src/adata/formats/json_data.py b/src/adata/formats/json_data.py index 8fb8c62..1082c79 100644 --- a/src/adata/formats/json_data.py +++ b/src/adata/formats/json_data.py @@ -9,6 +9,7 @@ from rich.console import Console from adata.core.read import decode_str_array +from adata.elements import spec from adata.formats.common import _check_json_exportable, _resolve from adata.elements.write import ( write_dense, @@ -78,6 +79,12 @@ def _pyify(value: Any, max_elements: int) -> Any: def _dataset_to_jsonable(ds: Any, max_elements: int) -> Any: + if spec.encoding_type(ds) == spec.NULL: + # A null is a placeholder -- an h5py.Empty or a 0-d zarr bool -- whose + # stored value is meaningless. Reading it would emit that placeholder + # rather than restoring the None it represents. + return None + if ds.shape == (): v = ds[()] return _pyify(v, max_elements=max_elements) @@ -181,7 +188,9 @@ def _write_json_list(parent: Any, name: str, value: list) -> None: return if arr is not None and arr.dtype.kind in ("U", "S", "O", "T"): - write_string_array(parent, name, arr.reshape(-1).tolist(), replace=True) + # Pass the shaped array through: flattening would turn a nested list + # such as [["a","b"],["c","d"]] into a length-4 vector. + write_string_array(parent, name, arr, replace=True) return write_scalar(parent, name, json.dumps(value), replace=True) diff --git a/src/adata/storage/__init__.py b/src/adata/storage/__init__.py index 929a95c..be84e77 100644 --- a/src/adata/storage/__init__.py +++ b/src/adata/storage/__init__.py @@ -26,6 +26,8 @@ class Store: root: Any path: Path zarr_format: Optional[int] = None + #: Whether to rewrite the Zarr consolidated metadata index on close. + consolidate: bool = False def close(self) -> None: if self.backend == "hdf5": @@ -33,6 +35,17 @@ def close(self) -> None: self.root.close() except Exception: return + return + + if self.consolidate and zarr is not None: + # anndata writes a consolidated metadata index at the root. New + # members land on disk regardless, but every reader that honours + # the index -- anndata included -- keeps using the stale copy and + # cannot see them, so the write looks like a silent no-op. + try: + zarr.consolidate_metadata(self.root.store, path=self.root.path) + except Exception: + return def __enter__(self) -> "Store": return self @@ -117,11 +130,19 @@ def open_store( backend = detect_backend(path) if backend == "zarr": _require_zarr() - kwargs = {} + kwargs: dict = {} if zarr_format is not None: kwargs["zarr_format"] = zarr_format + + writable = _is_writable_mode(mode) + if writable: + # Work against the real hierarchy: a consolidated index is a + # snapshot, so members added through it are invisible even to the + # handle that created them. + kwargs["use_consolidated"] = False + root = zarr.open_group(str(path), mode=mode, **kwargs) - if _is_writable_mode(mode): + if writable: ensure_anndata_root_attrs(root) elif require_anndata: warn_if_missing_anndata_root_attrs(root, path=path) @@ -130,6 +151,7 @@ def open_store( root=root, path=path, zarr_format=zarr_format_of(root), + consolidate=writable, ) root = h5py.File(path, mode) if _is_writable_mode(mode): diff --git a/tests/test_anndata_roundtrip.py b/tests/test_anndata_roundtrip.py index f6f7fab..de5680c 100644 --- a/tests/test_anndata_roundtrip.py +++ b/tests/test_anndata_roundtrip.py @@ -10,6 +10,7 @@ from __future__ import annotations +import json from pathlib import Path import numpy as np @@ -281,3 +282,126 @@ def test_sparse_subset_matches_scipy(tmp_path, layout): assert got.shape == (len(keep_obs), len(keep_var)) assert abs(got.X - expected).nnz == 0 assert type(got.X).__name__ == f"{layout}_matrix" + + +# --------------------------------------------------------------------------- +# regressions from review of #6 + + +@pytest.mark.parametrize("fmt", ["h5ad", "zarr"]) +def test_json_import_keeps_nested_string_arrays_rectangular(tmp_path, fmt): + """A nested string list was flattened into a vector on import.""" + store = tmp_path / f"base.{fmt}" + obj = ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=["c1", "c2"]), + var=pd.DataFrame(index=["g1", "g2"]), + ) + obj.write_zarr(store) if fmt == "zarr" else obj.write_h5ad(store) + + payload = tmp_path / "nested.json" + payload.write_text('{"grid": [["a","b"],["c","d"]], "nums": [[1,2],[3,4]]}') + + result = runner.invoke( + app, ["import", "dict", str(store), "uns/t", str(payload), "--inplace"] + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + uns = _read(store).uns["t"] + assert uns["grid"].shape == (2, 2), "a 2x2 string grid must not flatten" + assert uns["nums"].shape == (2, 2) + assert [list(row) for row in uns["grid"]] == [["a", "b"], ["c", "d"]] + + +@pytest.mark.parametrize("fmt", ["h5ad", "zarr"]) +def test_json_null_survives_a_full_round_trip(tmp_path, fmt): + """Exporting a `null` element emitted its storage placeholder, not None. + + The placeholder differs per backend -- an h5py.Empty, or a 0-d zarr bool -- + so neither serialised back to JSON null. + """ + store = tmp_path / f"base.{fmt}" + obj = ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=["c1", "c2"]), + var=pd.DataFrame(index=["g1", "g2"]), + ) + obj.write_zarr(store) if fmt == "zarr" else obj.write_h5ad(store) + + source = { + "grid": [["a", "b"], ["c", "d"]], + "nums": [[1, 2], [3, 4]], + "nothing": None, + "title": "run", + } + payload = tmp_path / "payload.json" + payload.write_text(json.dumps(source)) + + assert runner.invoke( + app, ["import", "dict", str(store), "uns/t", str(payload), "--inplace"] + ).exit_code == 0 + + out = tmp_path / "out.json" + result = runner.invoke( + app, ["export", "dict", str(store), "uns/t", "-o", str(out)] + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + assert json.loads(out.read_text()) == source, "JSON must round-trip exactly" + assert _read(store).uns["t"]["nothing"] is None + + +def test_writes_into_a_consolidated_zarr_store_are_visible(tmp_path): + """Edits to an anndata-written .zarr reported success but vanished. + + anndata writes a consolidated metadata index at the root. New members do + land on disk, but every reader that honours the index -- anndata included + -- keeps reading the stale snapshot, so the write looks like a no-op. + Stores the CLI writes itself are not consolidated, which is why this only + showed up against anndata's output. + """ + zarr = pytest.importorskip("zarr") + + store = tmp_path / "base.zarr" + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=["c1", "c2"]), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_zarr(store) + + root = json.loads((store / "zarr.json").read_text()) + assert root.get("consolidated_metadata"), "fixture must be consolidated" + + payload = tmp_path / "p.json" + payload.write_text('{"answer": 42}') + result = runner.invoke( + app, ["import", "dict", str(store), "uns/t", str(payload), "--inplace"] + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + # Visible through the consolidated index, not just on disk. + assert "t" in list(zarr.open_group(str(store))["uns"].keys()) + assert int(ad.read_zarr(store).uns["t"]["answer"]) == 42 + + +def test_subset_of_a_consolidated_zarr_store_round_trips(tmp_path): + """The same staleness would affect any command that writes a .zarr.""" + store = tmp_path / "base.zarr" + ad.AnnData( + X=np.ones((4, 2), dtype="float32"), + obs=pd.DataFrame(index=[f"c{i}" for i in range(4)]), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_zarr(store) + + names = tmp_path / "keep.txt" + names.write_text("c0\nc2\n") + out = tmp_path / "sub.zarr" + + result = runner.invoke( + app, ["subset", str(store), "-o", str(out), "--obs", str(names)] + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + + got = ad.read_zarr(out) + assert got.shape == (2, 2) + assert list(got.obs_names) == ["c0", "c2"] From 782e61730c009f6c59a1a63aafdb4b4f126e5f8c Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 13:21:34 +0100 Subject: [PATCH 17/23] Test against six real anndata releases, and close the coverage gaps The suite checked this tool mostly against stores it built itself, which cannot catch what actually broke it twice: anndata changing how it writes a file. tests/test_anndata_versions.py now builds a reference store with each of anndata 0.8, 0.9, 0.10, 0.11, 0.12 and 0.13 -- each in its own uv-assembled environment with era-appropriate pins -- and checks that the CLI reads it and that what the CLI writes reopens in that same release. 204 cases across both formats, marked `integration` and run as a separate CI job. The pins are load-bearing: a modern pandas makes string columns a type the older releases cannot write, and pandas 1.x has no Python 3.12 wheels, so 0.8 and 0.9 build against 3.11. New unit modules cover the element layer against HDF5, Zarr v2 and Zarr v3 (test_elements.py), the storage layer including cross-version copying (test_storage.py), the file-format paths and their failure modes (test_formats.py), dimension validation for every axis-bearing path (test_validate.py), the command surfaces reached only indirectly (test_commands_coverage.py), and invariants that must hold for any data -- subsetting everything is the identity, split partitions exactly, concat undoes split (test_invariants.py). Fixed while writing them: Zarr v3 -> v2 copies failed because v3 codec objects were forwarded into a v2 array, which only accepts numcodecs -- the mirror of the filters bug fixed earlier. Codecs now only travel between stores of the same Zarr version. Also silenced zarr's warning about consolidated metadata at our own call site, since writing it is deliberate. Adds a shared multi-backend store fixture, replacing the hand-rolled builders new tests would otherwise have copied, and docs/TESTING.md. 741 unit tests plus 204 compatibility tests, up from 244. Coverage 85.5% -> 92.4%, with a 90% floor enforced in CI. Co-Authored-By: Claude Opus 5 --- .github/workflows/tests.yml | 27 +- README.md | 1 + docs/TESTING.md | 81 ++++ docs/index.md | 6 +- pytest.ini | 2 +- src/adata/storage/__init__.py | 63 +-- tests/conftest.py | 78 +++- tests/fixtures/write_reference_store.py | 104 +++++ tests/reference_stores.py | 118 ++++++ tests/test_anndata_versions.py | 304 +++++++++++++++ tests/test_commands_coverage.py | 360 +++++++++++++++++ tests/test_elements.py | 437 +++++++++++++++++++++ tests/test_formats.py | 468 ++++++++++++++++++++++ tests/test_invariants.py | 493 ++++++++++++++++++++++++ tests/test_storage.py | 307 +++++++++++++++ tests/test_validate.py | 121 ++++++ 16 files changed, 2941 insertions(+), 29 deletions(-) create mode 100644 docs/TESTING.md create mode 100644 tests/fixtures/write_reference_store.py create mode 100644 tests/reference_stores.py create mode 100644 tests/test_anndata_versions.py create mode 100644 tests/test_commands_coverage.py create mode 100644 tests/test_elements.py create mode 100644 tests/test_formats.py create mode 100644 tests/test_invariants.py create mode 100644 tests/test_storage.py create mode 100644 tests/test_validate.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f29e48c..4c5e22f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,11 +48,12 @@ jobs: - name: Run tests with coverage run: | - uv run pytest -v -W default tests/ \ + uv run pytest -v -W default tests/ -m "not integration" \ --cov=adata \ --cov-report=term-missing \ --cov-report=xml \ --cov-report=html \ + --cov-fail-under=90 \ --junitxml=pytest-results-py${{ matrix.python-version }}.xml - name: Publish test results summary @@ -79,3 +80,27 @@ jobs: with: files: coverage.xml fail_ci_if_error: false + + # Builds a store with each pinned anndata release and checks the CLI against + # it. Split out because it assembles six environments with uv, which needs + # the network and is far slower than the unit suite. + compatibility: + runs-on: ubuntu-latest + timeout-minutes: 30 + name: anndata 0.8-0.13 compatibility + + permissions: + contents: read + + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + + - name: Install dependencies (frozen) + run: uv sync --extra dev --frozen + + - name: Run compatibility tests + run: uv run pytest -v -W default tests/test_anndata_versions.py -m integration diff --git a/README.md b/README.md index 7fd0157..30869c0 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample - [Get started](docs/GET_STARTED.md) — a short tutorial - [Command reference](docs/COMMANDS.md) — every command and flag - [Element spec: HDF5](docs/ELEMENTS_h5ad.md) / [Zarr](docs/ELEMENTS_zarr.md) — the on-disk format, and what this tool does with it +- [Testing](docs/TESTING.md) — how the suite is organised, and how compatibility is verified against six anndata releases - [Changelog](CHANGELOG.md) ## Docker diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 0000000..7159a14 --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,81 @@ +# Testing + +```bash +uv sync --extra dev +uv run pytest # everything +uv run pytest -m "not integration" # fast: no environment building, ~40s +uv run pytest -m integration # compatibility across anndata releases +``` + +CI runs the unit suite on Python 3.12 and 3.13 with a **90% coverage floor**, +and the compatibility suite as a separate job. + +## How the suite is organised + +| File | What it covers | +|---|---| +| `test_elements.py` | The element layer — encodings, string dtypes, readers and writers — run against HDF5, Zarr v2 and Zarr v3 | +| `test_storage.py` | Backend detection, copying, Zarr versions, consolidated metadata | +| `test_formats.py` | `.npy`, `.mtx`, image and JSON export/import, including their failure modes | +| `test_validate.py` | Dimension validation for every axis-bearing path, including `raw/` | +| `test_invariants.py` | Relationships that must hold for any data: subsetting everything is the identity, split partitions exactly, concat undoes split | +| `test_anndata_roundtrip.py` | anndata writes the fixtures, reads back our output | +| `test_anndata_versions.py` | Compatibility with six real anndata releases (see below) | +| `test_commands_phase2.py`, `test_commands_coverage.py`, `test_cli.py` | Command surfaces and error paths | +| `test_subset.py`, `test_export.py`, `test_import.py`, `test_info_read.py`, `test_zarr.py`, `test_query.py` | Per-feature unit tests | + +## Writing a test + +Use the `new_store` fixture to get a store on each backend in turn — it is +parametrised over `h5ad`, `zarr2` and `zarr3`, so one test body covers all +three: + +```python +def test_something(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_string_array(root["uns"], "s", ["a", "b"]) + with opener("r") as root: # reopen: Zarr rewrites its index on close + assert read_str_all(root["uns"]["s"]) == ["a", "b"] +``` + +For CLI surfaces, use the module-level `CliRunner` and assert on +`result.stdout + (result.stderr or "")`, since status goes to stderr. Strip +ANSI before matching message text — Rich also wraps long lines, so collapse +whitespace. + +## Compatibility testing against real anndata releases + +`test_anndata_versions.py` does not trust this repo's idea of the format. For +each release below it builds an environment with `uv`, writes a reference +store with that exact anndata, and then checks that the CLI reads it, and that +what the CLI writes can be reopened **by that same release**. + +| Release | Interpreter | Pins | Why it is in the list | +|---|---|---|---| +| 0.8.0 | 3.11 | `pandas<2`, `numpy<2`, `zarr<3` | Introduced `encoding-type`/`encoding-version` | +| 0.9.2 | 3.11 | `pandas<2`, `numpy<2`, `zarr<3` | | +| 0.10.9 | 3.12 | `pandas<3`, `numpy<2`, `zarr<3` | | +| 0.11.4 | 3.12 | `pandas<3`, `zarr<3` | The index became a `nullable-string-array` group | +| 0.12.2 | 3.12 | `pandas<3`, `zarr>=3` | Zarr v3 | +| 0.13.3 | 3.12 | `zarr>=3` | Zarr v3 by default | + +The pins matter: a modern pandas makes string columns a type the older +releases cannot write, and pandas 1.x has no wheels for Python 3.12. To add a +release, append to `RELEASES` in `tests/reference_stores.py`. + +First run downloads those environments; afterwards `uv` serves them from cache +and the suite takes about a minute. To skip it without the marker: + +```bash +ADATA_SKIP_VERSION_FIXTURES=1 uv run pytest +``` + +## Coverage + +```bash +uv run pytest -m "not integration" --cov=adata --cov-report=term-missing +``` + +The floor is 90%. What remains uncovered is mostly defensive `except` branches +around backend calls that do not fail in practice. diff --git a/docs/index.md b/docs/index.md index 1c193dd..67e12ee 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,6 +31,8 @@ docker run --rm -it -v /path/to/data:/data \ out in `.h5ad`, and what this tool does with it. - **[Element spec: Zarr](ELEMENTS_zarr.md)** — the same for `.zarr`, including the v2/v3 differences. +- **[Testing](TESTING.md)** — how the suite is organised, and how compatibility + is verified against six real anndata releases. ## At a glance @@ -50,7 +52,9 @@ adata import sparse new.h5ad X counts.mtx --inplace ## Format support Every AnnData on-disk layout is readable, from anndata 0.7.x through the -current spec; everything written is in the current spec. Both backends are +current spec; everything written is in the current spec. This is verified in +CI against stores written by anndata 0.8, 0.9, 0.10, 0.11, 0.12 and 0.13, in +both formats. Both backends are supported in both directions, so `adata subset in.h5ad -o out.zarr` converts as it filters. diff --git a/pytest.ini b/pytest.ini index a2cb692..45b5a3c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,4 @@ python_functions = test_* addopts = -v --strict-markers --tb=short markers = slow: marks tests as slow (deselect with '-m "not slow"') - integration: marks tests as integration tests + integration: builds environments with uv and needs the network on first run diff --git a/src/adata/storage/__init__.py b/src/adata/storage/__init__.py index be84e77..e446202 100644 --- a/src/adata/storage/__init__.py +++ b/src/adata/storage/__init__.py @@ -43,7 +43,13 @@ def close(self) -> None: # the index -- anndata included -- keeps using the stale copy and # cannot see them, so the write looks like a silent no-op. try: - zarr.consolidate_metadata(self.root.store, path=self.root.path) + with warnings.catch_warnings(): + # zarr notes that consolidated metadata is not part of the + # v3 spec. We write it deliberately, because anndata does + # and because without it our writes are invisible to + # readers that trust the index. + warnings.simplefilter("ignore") + zarr.consolidate_metadata(self.root.store, path=self.root.path) except Exception: return @@ -253,42 +259,49 @@ def dataset_create_kwargs( kw["fillvalue"] = src.fillvalue if target_backend == "zarr" and is_zarr_array(src): src_zarr_format = getattr(getattr(src, "metadata", None), "zarr_format", None) - if src_zarr_format == 3: - compressors = None - try: - compressors = getattr(src, "compressors", None) - except Exception: - compressors = None - if compressors is not None: - kw["compressors"] = compressors - else: + same_version = src_zarr_format == _target_zarr_format(kw_target) + + # Codecs only travel between stores of the same Zarr version: v2 holds + # numcodecs objects, v3 holds its own codec classes, and neither + # accepts the other's. Across versions the target's default is used + # rather than a translation that fails at creation time. + if same_version: + if src_zarr_format == 3: + try: + compressors = getattr(src, "compressors", None) + except Exception: + compressors = None + if compressors is not None: + kw["compressors"] = compressors + else: + try: + compressor = getattr(src, "compressor", None) + except Exception: + compressor = None + if compressor is not None: + kw["compressor"] = compressor + try: - compressor = getattr(src, "compressor", None) + filters = getattr(src, "filters", None) except Exception: - compressor = None - if compressor is not None: - kw["compressor"] = compressor - try: - filters = getattr(src, "filters", None) - except Exception: - filters = None - if filters: - # A v2 string array carries VLenUTF8 in `filters`; a v3 array - # rejects it (`Expected an ArrayArrayCodec`) because its string - # dtype encodes variable length itself. - if not (_target_zarr_format(kw_target) == 3 and _is_string_src(src)): + filters = None + # A v2 string array carries VLenUTF8 in `filters`; the v3 string + # dtype encodes variable length itself and rejects it. + if filters and not _is_string_src(src): kw["filters"] = filters + try: shards = getattr(src, "shards", None) except Exception: shards = None - if shards is not None: + if shards is not None and _target_zarr_format(kw_target) == 3: kw["shards"] = shards + try: fill_value = getattr(src, "fill_value", None) except Exception: fill_value = None - if fill_value is not None: + if fill_value is not None and not _is_string_src(src): kw["fill_value"] = fill_value return kw diff --git a/tests/conftest.py b/tests/conftest.py index e3b710f..b650f2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ -"""Pytest configuration and fixtures for h5ad tests.""" +"""Pytest configuration and fixtures for adata tests.""" +from contextlib import contextmanager from pathlib import Path +from typing import Any, Optional, Sequence import tempfile import pytest import h5py @@ -222,3 +224,77 @@ def sample_legacy_v010_h5ad(temp_dir): f.create_dataset("X", data=X) return file_path + + +# --------------------------------------------------------------------------- +# Shared builders +# +# The fixtures above are hand-rolled h5py, one per shape of store. These +# builders exist so a test can ask for a store containing a particular element +# on a particular backend without repeating that construction, and so the same +# test body can run against HDF5, Zarr v2 and Zarr v3. + +from adata.elements import write as ew +from adata.storage import open_store + + +BACKENDS = ["h5ad", "zarr2", "zarr3"] + + +def backend_path(tmp_dir: Path, backend: str, stem: str = "store") -> Path: + """Path with the extension that makes `detect_backend` pick `backend`.""" + return tmp_dir / (f"{stem}.h5ad" if backend == "h5ad" else f"{stem}.zarr") + + +def backend_zarr_format(backend: str) -> Optional[int]: + return {"zarr2": 2, "zarr3": 3}.get(backend) + + +@pytest.fixture(params=BACKENDS) +def backend(request) -> str: + """Run a test once per storage backend, including both Zarr versions.""" + return request.param + + +@contextmanager +def open_new(path: Path, backend: str): + """Open a fresh store for writing on the given backend.""" + with open_store(path, "w", zarr_format=backend_zarr_format(backend)) as store: + yield store.root + + +def make_skeleton( + root: Any, + obs_names: Sequence[str] = ("c1", "c2", "c3"), + var_names: Sequence[str] = ("g1", "g2"), +) -> None: + """Write the obs/var frames and mapping groups every store needs.""" + ew.write_dataframe_header(root, "obs", list(obs_names), []) + ew.write_dataframe_header(root, "var", list(var_names), []) + ew.ensure_anndata_skeleton(root) + + +@pytest.fixture +def new_store(temp_dir, backend): + """Factory returning ``(path, opener)`` for a store on the current backend. + + The opener is a context manager so a test can write, close, and reopen -- + which matters on Zarr, where the consolidated metadata index is only + rewritten on close. + """ + + def _make(stem: str = "store"): + path = backend_path(temp_dir, backend, stem) + + @contextmanager + def _open(mode: str = "a"): + with open_store( + path, mode, zarr_format=backend_zarr_format(backend) + ) as store: + yield store.root + + with open_new(path, backend) as root: + make_skeleton(root) + return path, _open + + return _make diff --git a/tests/fixtures/write_reference_store.py b/tests/fixtures/write_reference_store.py new file mode 100644 index 0000000..3fef60e --- /dev/null +++ b/tests/fixtures/write_reference_store.py @@ -0,0 +1,104 @@ +"""Write a reference AnnData store, run inside a pinned anndata environment. + +Invoked by tests/reference_stores.py through `uv run --with anndata==`, so +this file must stay compatible with every anndata release under test (0.8 +onward) and must not import anything from `adata`. Features that only exist in +later versions are attempted and skipped rather than assumed. + +Usage: write_reference_store.py +""" + +from __future__ import annotations + +import sys +import warnings + +warnings.filterwarnings("ignore") + +import anndata as ad # noqa: E402 +import numpy as np # noqa: E402 +import pandas as pd # noqa: E402 +from scipy import sparse # noqa: E402 + +N_OBS, N_VAR = 6, 4 + + +def build(): + """An object exercising the encodings this tool has to understand.""" + obs = pd.DataFrame( + { + "cell_type": pd.Categorical( + ["A", "B", "A", "C", "B", "A"], + categories=["A", "B", "C"], + ordered=True, + ), + "unordered": pd.Categorical(["x", "y", "x", "y", "x", "y"]), + "n_counts": np.arange(N_OBS, dtype="int32"), + "score": np.linspace(0, 1, N_OBS).astype("float64"), + "free_text": ["a", "bb", "ccc", "d", "ee", "f"], + }, + index=[f"cell_{i}" for i in range(N_OBS)], + ) + + # Nullable dtypes have been supported since 0.8, but guard anyway. + try: + obs["nullable_int"] = pd.array([1, 2, None, 4, 5, None], dtype="Int32") + obs["nullable_bool"] = pd.array( + [True, False, None, True, False, None], dtype="boolean" + ) + except Exception: + pass + + var = pd.DataFrame( + { + "gene_ids": [f"ENSG{i}" for i in range(N_VAR)], + "highly_variable": [True, False, True, False], + }, + index=[f"gene_{i}" for i in range(N_VAR)], + ) + + X = sparse.csr_matrix( + np.random.default_rng(0).poisson(1.0, (N_OBS, N_VAR)).astype("float32") + ) + + obj = ad.AnnData(X=X, obs=obs, var=var) + obj.layers["counts"] = X.copy() + obj.layers["dense"] = np.asarray(X.todense()) + obj.obsm["X_pca"] = np.zeros((N_OBS, 3), dtype="float32") + obj.varm["PCs"] = np.zeros((N_VAR, 3), dtype="float32") + obj.obsp["connectivities"] = sparse.csr_matrix(np.eye(N_OBS, dtype="float32")) + obj.varp["corr"] = sparse.csr_matrix(np.eye(N_VAR, dtype="float32")) + + obj.uns["a_string"] = "hello" + obj.uns["an_int"] = 42 + obj.uns["a_float"] = 3.25 + obj.uns["a_bool"] = True + obj.uns["a_list"] = ["x", "y", "z"] + obj.uns["numbers"] = np.arange(5) + obj.uns["nested"] = {"deep": {"value": 1.5}} + + obj.raw = obj + return obj + + +def main(out_dir: str, fmt: str) -> None: + import os + from importlib.metadata import version + + os.makedirs(out_dir, exist_ok=True) + obj = build() + target = f"{out_dir}/reference.{fmt}" + if fmt == "zarr": + obj.write_zarr(target) + else: + obj.write_h5ad(target) + + try: + zarr_version = version("zarr") + except Exception: + zarr_version = "absent" + print(f"WROTE {target} anndata={version('anndata')} zarr={zarr_version}") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) diff --git a/tests/reference_stores.py b/tests/reference_stores.py new file mode 100644 index 0000000..d70925b --- /dev/null +++ b/tests/reference_stores.py @@ -0,0 +1,118 @@ +"""Build reference stores with real, pinned anndata releases. + +The point is to check compatibility against what anndata actually wrote at +each version, rather than against this repo's idea of the format. Each store +is produced by running `tests/fixtures/write_reference_store.py` inside an +environment `uv` assembles for that release, so nothing here depends on the +versions installed for the test suite itself. + +Stores are built once per session and cached, since assembling six +environments is slow the first time and free afterwards. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional + +WRITER = Path(__file__).parent / "fixtures" / "write_reference_store.py" + + +@dataclass(frozen=True) +class Release: + """An anndata release to build fixtures with.""" + + #: Requirement string passed to `uv --with`. + spec: str + #: Short label used in test ids. + label: str + #: Era-appropriate pins. Modern pandas makes string columns a type the + #: older releases cannot write, so each release needs the stack it shipped + #: against rather than today's. + extras: tuple = () + #: Interpreter to run under. pandas 1.x has no wheels for 3.12, so the + #: oldest releases build against 3.11. + python: str = "3.12" + + +#: anndata 0.8 introduced the encoding-type/encoding-version scheme, so it is +#: the oldest release whose output this tool claims to read. 0.11 is where the +#: index became a nullable-string-array group, and 0.12 is where Zarr v3 +#: arrived -- the two changes that broke the CLI. +RELEASES: List[Release] = [ + Release( + "anndata==0.8.0", "0.8", + extras=("pandas<2", "numpy<2", "zarr<3"), python="3.11", + ), + Release( + "anndata==0.9.2", "0.9", + extras=("pandas<2", "numpy<2", "zarr<3"), python="3.11", + ), + Release("anndata==0.10.9", "0.10", extras=("pandas<3", "numpy<2", "zarr<3")), + Release("anndata==0.11.4", "0.11", extras=("pandas<3", "zarr<3")), + Release("anndata==0.12.2", "0.12", extras=("pandas<3", "zarr>=3")), + Release("anndata~=0.13.3", "0.13", extras=("zarr>=3",)), +] + + +class ReferenceUnavailable(RuntimeError): + """Raised when a store could not be produced for a release.""" + + +def uv_available() -> bool: + return shutil.which("uv") is not None + + +def offline() -> bool: + """Honour the usual opt-out for tests that need to reach the network.""" + return os.environ.get("ADATA_SKIP_VERSION_FIXTURES", "").strip() not in ("", "0") + + +def build(release: Release, fmt: str, out_dir: Path) -> Path: + """Write one reference store, returning its path.""" + out_dir.mkdir(parents=True, exist_ok=True) + cmd = [ + "uv", "run", "--no-project", "--python", release.python, + "--with", release.spec, "--with", "scipy", + ] + for extra in release.extras: + cmd += ["--with", extra] + cmd += [str(WRITER), str(out_dir), fmt] + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=900) + target = out_dir / f"reference.{fmt}" + if result.returncode != 0 or not target.exists(): + raise ReferenceUnavailable( + f"{release.spec} ({fmt}) could not be built:\n" + f"{result.stdout[-2000:]}\n{result.stderr[-2000:]}" + ) + return target + + +class ReferenceCache: + """Builds each (release, format) store once and remembers failures.""" + + def __init__(self, root: Path) -> None: + self.root = root + self._built: Dict[str, Path] = {} + self._failed: Dict[str, str] = {} + + def get(self, release: Release, fmt: str) -> Optional[Path]: + key = f"{release.label}-{fmt}" + if key in self._built: + return self._built[key] + if key in self._failed: + raise ReferenceUnavailable(self._failed[key]) + + try: + path = build(release, fmt, self.root / key) + except Exception as exc: # noqa: BLE001 - reported to the test as a skip + self._failed[key] = str(exc) + raise ReferenceUnavailable(str(exc)) from exc + + self._built[key] = path + return path diff --git a/tests/test_anndata_versions.py b/tests/test_anndata_versions.py new file mode 100644 index 0000000..4bf5e8a --- /dev/null +++ b/tests/test_anndata_versions.py @@ -0,0 +1,304 @@ +"""Compatibility against stores written by real anndata releases, 0.8 to 0.13. + +The rest of the suite checks this tool against stores it built itself, which +cannot catch the thing that actually broke it: anndata changing how it writes +a file. Here each fixture is produced by running a pinned anndata release in +its own environment, so the assertions are about what those releases really +wrote rather than about this repo's understanding of the format. + +Marked `integration` because building the environments needs `uv` and, the +first time, the network: + + pytest -m integration # just these + pytest -m "not integration" # skip them + ADATA_SKIP_VERSION_FIXTURES=1 pytest # skip them via the environment +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +from typer.testing import CliRunner + +from adata.cli import app +from adata.core.info import axis_len, get_entry_type +from adata.elements import spec +from adata.elements.read import element_len, read_str_all, resolve_index +from adata.storage import is_group, open_store + +from tests.reference_stores import ( + RELEASES, + ReferenceCache, + ReferenceUnavailable, + offline, + uv_available, +) + +pytestmark = pytest.mark.integration + +runner = CliRunner() + +N_OBS, N_VAR = 6, 4 +OBS_NAMES = [f"cell_{i}" for i in range(N_OBS)] +VAR_NAMES = [f"gene_{i}" for i in range(N_VAR)] + + +@pytest.fixture(scope="session") +def cache(tmp_path_factory) -> ReferenceCache: + if offline(): + pytest.skip("ADATA_SKIP_VERSION_FIXTURES is set") + if not uv_available(): + pytest.skip("uv is needed to build the reference stores") + return ReferenceCache(tmp_path_factory.mktemp("anndata-versions")) + + +@pytest.fixture(params=RELEASES, ids=[r.label for r in RELEASES]) +def release(request): + return request.param + + +@pytest.fixture(params=["h5ad", "zarr"]) +def fmt(request) -> str: + return request.param + + +@pytest.fixture +def store(cache, release, fmt) -> Path: + """A reference store, or a skip explaining why it could not be built.""" + try: + return cache.get(release, fmt) + except ReferenceUnavailable as exc: + pytest.skip(f"could not build {release.label} ({fmt}): {exc}") + + +def _out(result) -> str: + return result.stdout + (result.stderr or "") + + +# --------------------------------------------------------------------------- +# reading + + +def test_view_reports_the_right_shape(store): + result = runner.invoke(app, ["view", str(store)]) + assert result.exit_code == 0, _out(result) + assert f"{N_OBS} × {N_VAR}" in result.stdout + + +def test_ls_walks_the_whole_store(store): + result = runner.invoke(app, ["ls", str(store), "--long"]) + assert result.exit_code == 0, _out(result) + for key in ("obs", "var", "X", "uns", "layers"): + assert key in result.stdout + + +def test_axis_lengths_are_readable_whatever_the_index_layout(store): + """The index is a dataset in older releases and a group from 0.11.""" + with open_store(store, "r") as handle: + assert axis_len(handle.root, "obs") == N_OBS + assert axis_len(handle.root, "var") == N_VAR + + +def test_index_names_read_back_correctly(store): + with open_store(store, "r") as handle: + for axis, expected in (("obs", OBS_NAMES), ("var", VAR_NAMES)): + index, _ = resolve_index(handle.root[axis], axis) + assert read_str_all(index) == expected + assert element_len(index) == len(expected) + + +def test_every_obs_column_exports(store): + result = runner.invoke(app, ["export", "dataframe", str(store), "obs"]) + assert result.exit_code == 0, _out(result) + + lines = [ln for ln in result.stdout.splitlines() if ln] + header = lines[0].split(",") + assert len(lines) == N_OBS + 1 + for column in ("cell_type", "n_counts", "score", "free_text"): + assert column in header, f"{column} missing from {header}" + + # The categorical must render as its labels, not its codes. + cell_type = lines[1].split(",")[header.index("cell_type")] + assert cell_type == "A" + + +def test_nullable_columns_render_missing_values_as_empty(store): + result = runner.invoke(app, ["export", "dataframe", str(store), "obs"]) + assert result.exit_code == 0, _out(result) + + lines = [ln for ln in result.stdout.splitlines() if ln] + header = lines[0].split(",") + if "nullable_int" not in header: + pytest.skip("this release did not write a nullable column") + column = header.index("nullable_int") + assert lines[3].split(",")[column] == "", "row 3 is NA in the fixture" + + +def test_raw_var_exports_from_its_own_path(store): + result = runner.invoke(app, ["export", "dataframe", str(store), "raw/var"]) + assert result.exit_code == 0, _out(result) + assert "gene_ids" in result.stdout + + +def test_sparse_x_exports_to_matrix_market(store): + result = runner.invoke(app, ["export", "sparse", str(store), "X"]) + assert result.exit_code == 0, _out(result) + lines = result.stdout.splitlines() + assert lines[0].startswith("%%MatrixMarket") + dims = [ln for ln in lines if not ln.startswith("%")][0].split() + assert dims[:2] == [str(N_OBS), str(N_VAR)] + + +def test_uns_exports_to_json(store): + import json + + result = runner.invoke(app, ["export", "dict", str(store), "uns"]) + assert result.exit_code == 0, _out(result) + payload = json.loads(result.stdout) + assert payload["a_string"] == "hello" + assert int(payload["an_int"]) == 42 + assert payload["nested"]["deep"]["value"] == pytest.approx(1.5) + + +def test_categorical_is_detected_whatever_the_layout(store): + """0.7-era categoricals are codes plus a reference; later ones are groups.""" + with open_store(store, "r") as handle: + info = get_entry_type(handle.root["obs"]["cell_type"]) + assert info["type"] == "categorical" + + +def test_encodings_are_recognised_not_guessed(store): + with open_store(store, "r") as handle: + root = handle.root + assert get_entry_type(root["X"])["type"] == "sparse-matrix" + assert get_entry_type(root["obs"])["type"] == "dataframe" + assert get_entry_type(root["obsm"]["X_pca"])["type"] in ( + "array", + "dense-matrix", + ) + + +# --------------------------------------------------------------------------- +# writing, checked by reading the result back with the same anndata release + + +@pytest.fixture +def read_back(release, fmt): + """Read a store using the same anndata release that wrote the fixture. + + Checking our output with the *current* anndata would not show whether an + older release can still open what we wrote. + """ + import subprocess + + from tests.reference_stores import Release + + def _read(path: Path, expression: str): + script = ( + "import warnings; warnings.filterwarnings('ignore')\n" + "import anndata as ad, sys\n" + f"a = ad.read_{'zarr' if fmt == 'zarr' else 'h5ad'}(sys.argv[1])\n" + f"print('RESULT', {expression})\n" + ) + cmd = [ + "uv", "run", "--no-project", "--python", release.python, + "--with", release.spec, "--with", "scipy", + ] + for extra in release.extras: + cmd += ["--with", extra] + cmd += ["python", "-c", script, str(path)] + out = subprocess.run(cmd, capture_output=True, text=True, timeout=900) + line = [ + ln for ln in out.stdout.splitlines() if ln.startswith("RESULT") + ] + if not line: + pytest.fail(f"reading back failed:\n{out.stdout}\n{out.stderr}") + return line[0][len("RESULT ") :] + + return _read + + +def test_subset_output_opens_in_the_release_that_wrote_the_input( + store, tmp_path, fmt, read_back +): + """A store we write must be readable by the anndata that made the source.""" + names = tmp_path / "keep.txt" + names.write_text("cell_0\ncell_2\n") + out = tmp_path / f"subset.{fmt}" + + result = runner.invoke( + app, ["subset", str(store), "-o", str(out), "--obs", str(names)] + ) + assert result.exit_code == 0, _out(result) + + assert read_back(out, "(a.n_obs, a.n_vars)") == "(2, 4)" + assert read_back(out, "list(a.obs_names)") == "['cell_0', 'cell_2']" + assert read_back(out, "str(a.obs['cell_type'].dtype)") == "category" + + +def test_subset_preserves_raw_across_versions(store, tmp_path, fmt, read_back): + names = tmp_path / "keep.txt" + names.write_text("cell_0\ncell_2\n") + out = tmp_path / f"subset.{fmt}" + assert runner.invoke( + app, ["subset", str(store), "-o", str(out), "--obs", str(names)] + ).exit_code == 0 + + assert read_back(out, "a.raw is not None") == "True" + assert read_back(out, "(a.raw.shape)") == "(2, 4)" + + +@pytest.mark.parametrize("target", ["h5ad", "zarr"]) +def test_conversion_between_backends_keeps_the_data(store, tmp_path, target): + """Reading with the current anndata is enough to prove the crossing works.""" + ad = pytest.importorskip("anndata") + + names = tmp_path / "keep.txt" + names.write_text("\n".join(OBS_NAMES)) + out = tmp_path / f"converted.{target}" + + result = runner.invoke( + app, ["subset", str(store), "-o", str(out), "--obs", str(names)] + ) + assert result.exit_code == 0, _out(result) + + got = ad.read_zarr(out) if target == "zarr" else ad.read_h5ad(out) + assert got.shape == (N_OBS, N_VAR) + assert list(got.obs_names) == OBS_NAMES + assert str(got.obs["cell_type"].dtype) == "category" + assert sorted(got.obs["cell_type"].cat.categories) == ["A", "B", "C"] + + +def test_split_then_concat_round_trips_across_versions(store, tmp_path, fmt): + ad = pytest.importorskip("anndata") + + parts = tmp_path / "parts" + assert runner.invoke( + app, ["split", str(store), "--by", "cell_type", "-o", str(parts)] + ).exit_code == 0 + + pieces = sorted(str(p) for p in parts.glob(f"*.{fmt}")) + assert len(pieces) == 3, f"expected one store per category, got {pieces}" + + merged = tmp_path / f"merged.{fmt}" + result = runner.invoke(app, ["concat", *pieces, "-o", str(merged)]) + assert result.exit_code == 0, _out(result) + + got = ad.read_zarr(merged) if fmt == "zarr" else ad.read_h5ad(merged) + assert got.shape == (N_OBS, N_VAR) + assert sorted(got.obs_names) == sorted(OBS_NAMES) + + +def test_query_filtering_works_on_every_version(store, tmp_path, fmt): + ad = pytest.importorskip("anndata") + + out = tmp_path / f"filtered.{fmt}" + result = runner.invoke( + app, ["subset", str(store), "-o", str(out), "-q", "cell_type == A"] + ) + assert result.exit_code == 0, _out(result) + + got = ad.read_zarr(out) if fmt == "zarr" else ad.read_h5ad(out) + assert list(got.obs_names) == ["cell_0", "cell_2", "cell_5"] diff --git a/tests/test_commands_coverage.py b/tests/test_commands_coverage.py new file mode 100644 index 0000000..2cd1a88 --- /dev/null +++ b/tests/test_commands_coverage.py @@ -0,0 +1,360 @@ +"""Tests for command surfaces the rest of the suite reaches only indirectly. + +Covers the extension-dispatching `import_object` entry point, every `ls` +output mode, the concat merge strategies, and the CLI's error paths -- the +places where a user gets a message rather than a traceback. +""" + +from __future__ import annotations + +import json +import re + +import numpy as np +import pytest +from rich.console import Console +from typer.testing import CliRunner + +from adata.cli import app +from adata.commands.import_data import import_object +from adata.commands.ls import list_store, walk +from adata.core.concat import MERGE_STRATEGIES, _merge_values, _MISSING +from adata.elements import spec +from adata.elements import write as ew +from adata.storage import open_store + +runner = CliRunner() +console = Console(stderr=True) +_ANSI = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]") + + +def _out(result) -> str: + return " ".join(_ANSI.sub("", result.stdout + (result.stderr or "")).split()) + + +@pytest.fixture +def store(new_store): + path, opener = new_store() + return path, opener + + +# --------------------------------------------------------------------------- +# import_object: dispatch by file extension + + +@pytest.mark.parametrize( + "suffix,write,target", + [ + (".csv", lambda p: p.write_text("_index,v\nc1,1\nc2,2\nc3,3\n"), "obs"), + (".npy", lambda p: np.save(p, np.zeros((3, 2))), "obsm/a"), + (".json", lambda p: p.write_text('{"k": 1}'), "uns/j"), + ], +) +def test_import_object_dispatches_on_extension( + store, temp_dir, suffix, write, target +): + path, opener = store + source = temp_dir / f"in{suffix}" + write(source) + + import_object( + file=path, + obj=target, + input_file=source, + output_file=None, + inplace=True, + index_column=None, + console=console, + ) + + with opener("r") as root: + node = root + for part in target.split("/"): + node = node[part] + assert node is not None + + +def test_import_object_dispatches_mtx(store, temp_dir): + path, opener = store + source = temp_dir / "in.mtx" + source.write_text( + "%%MatrixMarket matrix coordinate real general\n3 2 2\n1 1 1.0\n2 2 2.0\n" + ) + import_object( + file=path, obj="X", input_file=source, output_file=None, + inplace=True, index_column=None, console=console, + ) + with opener("r") as root: + assert spec.encoding_type(root["X"]) == spec.CSR_MATRIX + + +def test_import_object_rejects_an_unknown_extension(store, temp_dir): + path, _ = store + source = temp_dir / "in.parquet" + source.write_text("x") + with pytest.raises(ValueError, match="Unsupported input file extension"): + import_object( + file=path, obj="obs", input_file=source, output_file=None, + inplace=True, index_column=None, console=console, + ) + + +def test_import_object_rejects_index_column_for_the_wrong_format(store, temp_dir): + path, _ = store + source = temp_dir / "in.npy" + np.save(source, np.zeros((3, 2))) + with pytest.raises(ValueError, match="--index-column is only valid"): + import_object( + file=path, obj="obsm/a", input_file=source, output_file=None, + inplace=True, index_column="c", console=console, + ) + + +def test_import_object_requires_an_output_when_not_inplace(store, temp_dir): + path, _ = store + source = temp_dir / "in.json" + source.write_text("{}") + with pytest.raises(ValueError, match="Output file is required"): + import_object( + file=path, obj="uns/j", input_file=source, output_file=None, + inplace=False, index_column=None, console=console, + ) + + +def test_import_to_an_output_copies_the_source_first(store, temp_dir, backend): + """Non-inplace imports must not touch the input store.""" + from tests.conftest import backend_path + + path, opener = store + source = temp_dir / "in.json" + source.write_text('{"k": 1}') + out = backend_path(temp_dir, backend, "out") + + import_object( + file=path, obj="uns/j", input_file=source, output_file=out, + inplace=False, index_column=None, console=console, + ) + + with open_store(out, "r") as handle: + assert "j" in handle.root["uns"] + with opener("r") as root: + assert "j" not in root["uns"], "the source must be left alone" + + +# --------------------------------------------------------------------------- +# ls + + +@pytest.fixture +def populated(store): + path, opener = store + with opener("a") as root: + ew.write_dense(root["obsm"], "X_pca", np.zeros((3, 2))) + group = ew.write_mapping(root["uns"], "spatial") + ew.write_scalar(group, "scale", 1.5) + return path + + +def test_ls_tree_lists_every_member(populated, capsys): + list_store(populated, Console()) + out = capsys.readouterr().out + for key in ("obs", "var", "obsm", "X_pca", "spatial"): + assert key in out + + +def test_ls_long_shows_types_and_shapes(populated, capsys): + list_store(populated, Console(width=200), long=True) + out = capsys.readouterr().out + assert "dataframe" in out + assert "(3, 2)" in out + + +def test_ls_plain_emits_bare_paths(populated, capsys): + list_store(populated, Console(), plain=True) + lines = [ln for ln in capsys.readouterr().out.splitlines() if ln] + assert "obsm/X_pca" in lines + assert all(" " not in ln for ln in lines), "plain output must pipe cleanly" + + +def test_ls_can_start_below_a_path(populated, capsys): + list_store(populated, Console(), entry_path="obsm", plain=True) + lines = [ln for ln in capsys.readouterr().out.splitlines() if ln] + assert lines == ["obsm/X_pca"] + + +def test_ls_of_a_dataset_shows_just_that_dataset(populated, capsys): + list_store(populated, Console(width=200), entry_path="obsm/X_pca", long=True) + out = capsys.readouterr().out + assert "X_pca" in out + + +def test_ls_plain_of_a_dataset_prints_its_path(populated, capsys): + list_store(populated, Console(), entry_path="obsm/X_pca", plain=True) + assert capsys.readouterr().out.strip() == "obsm/X_pca" + + +def test_ls_depth_limits_recursion(populated, capsys): + list_store(populated, Console(), depth=1, plain=True) + lines = [ln for ln in capsys.readouterr().out.splitlines() if ln] + assert "obsm" in lines + assert "obsm/X_pca" not in lines + + +def test_ls_reports_a_missing_path(populated): + with pytest.raises(KeyError, match="not found"): + list_store(populated, Console(), entry_path="nope") + + +def test_walk_yields_nothing_for_a_dataset(populated): + with open_store(populated, "r") as handle: + assert list(walk(handle.root["obsm"]["X_pca"])) == [] + + +# --------------------------------------------------------------------------- +# merge strategies + + +@pytest.mark.parametrize( + "strategy,values,expected", + [ + (None, ["a", "a"], (False, None)), + ("same", ["a", "a"], (True, "a")), + ("same", ["a", "b"], (False, None)), + ("same", ["a", _MISSING], (False, None)), + ("unique", ["a", "a"], (True, "a")), + ("unique", ["a", _MISSING], (True, "a")), + ("unique", ["a", "b"], (False, None)), + ("first", ["a", "b"], (True, "a")), + ("first", [_MISSING, "b"], (True, "b")), + ("only", ["a", _MISSING], (True, "a")), + ("only", ["a", "b"], (False, None)), + ("only", ["a", "a"], (False, None)), + ("same", [_MISSING, _MISSING], (False, None)), + ], +) +def test_merge_strategies(strategy, values, expected): + assert _merge_values(list(values), strategy) == expected + + +def test_unknown_merge_strategy_is_reported(): + with pytest.raises(ValueError, match="Unknown merge strategy"): + _merge_values(["a"], "bogus") + + +def test_documented_strategies_all_work(): + for strategy in MERGE_STRATEGIES: + _merge_values(["a", "a"], strategy) + + +# --------------------------------------------------------------------------- +# CLI error paths + + +def test_view_reports_a_missing_entry(new_store): + path, _ = new_store() + result = runner.invoke(app, ["view", str(path), "nope"]) + assert "not found" in _out(result) + + +def test_export_reports_a_missing_entry(new_store): + path, _ = new_store() + result = runner.invoke(app, ["export", "dict", str(path), "nope"]) + assert result.exit_code == 1 + assert "not found" in _out(result) + + +def test_import_reports_a_missing_output_flag(new_store, temp_dir): + path, _ = new_store() + source = temp_dir / "in.json" + source.write_text("{}") + for sub, args in ( + ("dict", ["uns/j", str(source)]), + ("array", ["obsm/a", str(source)]), + ("sparse", ["X", str(source)]), + ("image", ["uns/i", str(source)]), + ): + result = runner.invoke(app, ["import", sub, str(path), *args]) + assert result.exit_code == 1 + assert "Output file is required" in _out(result) + + +def test_import_dataframe_reports_a_missing_output_flag(new_store, temp_dir): + path, _ = new_store() + source = temp_dir / "in.csv" + source.write_text("a\n1\n") + result = runner.invoke(app, ["import", "dataframe", str(path), "obs", str(source)]) + assert result.exit_code == 1 + assert "Output file is required" in _out(result) + + +def test_create_rejects_a_bad_zarr_format(temp_dir): + result = runner.invoke( + app, + ["create", str(temp_dir / "x.zarr"), "--n-obs", "2", "--n-var", "2", + "--zarr-format", "4"], + ) + assert result.exit_code == 1 + assert "must be 2 or 3" in _out(result) + + +def test_create_requires_a_size_or_a_name_file(temp_dir): + result = runner.invoke(app, ["create", str(temp_dir / "x.h5ad"), "--n-var", "2"]) + assert result.exit_code == 1 + assert "--n-obs" in _out(result) + + +def test_create_rejects_a_negative_size(temp_dir): + result = runner.invoke( + app, ["create", str(temp_dir / "x.h5ad"), "--n-obs", "-1", "--n-var", "2"] + ) + assert result.exit_code == 1 + assert "must not be negative" in _out(result) + + +def test_create_rejects_duplicate_names(temp_dir): + names = temp_dir / "dup.txt" + names.write_text("a\na\n") + result = runner.invoke( + app, + ["create", str(temp_dir / "x.h5ad"), "--obs-names", str(names), "--n-var", "2"], + ) + assert result.exit_code == 1 + assert "duplicate names" in _out(result) + + +def test_create_rejects_an_empty_name_file(temp_dir): + names = temp_dir / "empty.txt" + names.write_text("\n\n") + result = runner.invoke( + app, + ["create", str(temp_dir / "x.h5ad"), "--obs-names", str(names), "--n-var", "2"], + ) + assert result.exit_code == 1 + assert "no names" in _out(result) + + +def test_split_rejects_a_bad_axis(new_store, temp_dir): + path, _ = new_store() + result = runner.invoke( + app, ["split", str(path), "--by", "x", "-o", str(temp_dir / "o"), "--axis", "z"] + ) + assert result.exit_code == 1 + assert "'obs' or 'var'" in _out(result) + + +def test_ls_reports_a_missing_store(): + result = runner.invoke(app, ["ls", "does-not-exist.h5ad"]) + assert result.exit_code != 0 + + +def test_version_flag_prints_only_the_version(): + result = runner.invoke(app, ["--version"]) + assert result.exit_code == 0 + assert re.fullmatch(r"\d+\.\d+\.\d+\S*", result.stdout.strip()) + + +def test_norm_path_rejects_an_empty_path(): + from adata.util.path import norm_path + + with pytest.raises(ValueError, match="non-empty"): + norm_path(" ") diff --git a/tests/test_elements.py b/tests/test_elements.py new file mode 100644 index 0000000..525d19d --- /dev/null +++ b/tests/test_elements.py @@ -0,0 +1,437 @@ +"""Unit tests for the element layer, across HDF5, Zarr v2 and Zarr v3. + +Everything else in the codebase reads and writes through these functions, so +a gap here shows up as a subtle corruption several layers away. Each test runs +against all three backends, because the two most expensive bugs in this +project's history -- variable-length strings and group-valued indices -- were +both cases where one backend behaved differently from the other. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from adata.elements import spec +from adata.elements import write as ew +from adata.elements.read import ( + dataframe_columns, + decode_str_array, + element_len, + is_ordered, + read_categories, + read_masked_column, + read_str_all, + read_str_chunk, + resolve_index, +) +from adata.elements.strings import ( + as_str_array, + is_string_dtype, + is_string_element, + string_dtype_for, + target_dtype, +) +from adata.storage import is_dataset, is_group + + +# --------------------------------------------------------------------------- +# spec + + +def test_every_encoding_has_a_declared_version(): + """A type missing from the table would be written without a version.""" + names = { + v + for k, v in vars(spec).items() + if k.isupper() and isinstance(v, str) and not k.endswith("_TYPES") + } + for name in names: + if name in spec.CURRENT_VERSION: + assert spec.CURRENT_VERSION[name].count(".") == 2 + + +def test_decode_attr_normalises_both_backends_spellings(): + assert spec.decode_attr(b"dataframe") == "dataframe" + assert spec.decode_attr("dataframe") == "dataframe" + assert spec.decode_attr(np.str_("x")) == "x" + assert spec.decode_attr(np.True_) is True + assert spec.decode_attr(None) is None + + +def test_encoding_of_reports_none_for_untagged_elements(): + class Bare: + attrs: dict = {} + + assert spec.encoding_of(Bare()) == (None, None) + assert spec.encoding_type(Bare()) is None + + +# --------------------------------------------------------------------------- +# strings + + +@pytest.mark.parametrize( + "dtype,expected", + [ + (np.dtype("S5"), True), + (np.dtype(" read, on every backend + + +def test_string_array_round_trips(new_store): + path, opener = new_store() + values = ["alpha", "", "café", "a longer one"] + with opener("a") as root: + ew.write_string_array(root["uns"], "s", values) + with opener("r") as root: + ds = root["uns"]["s"] + assert spec.encoding_of(ds) == (spec.STRING_ARRAY, "0.2.0") + assert read_str_all(ds) == values + assert is_string_element(ds) + + +def test_string_array_keeps_multidimensional_shape(new_store): + path, opener = new_store() + grid = np.array([["a", "b"], ["c", "d"]], dtype=object) + with opener("a") as root: + ew.write_string_array(root["uns"], "grid", grid) + with opener("r") as root: + assert root["uns"]["grid"].shape == (2, 2) + + +@pytest.mark.parametrize("ordered", [True, False]) +def test_categorical_round_trips_with_order(new_store, ordered): + path, opener = new_store() + with opener("a") as root: + ew.write_categorical( + root["uns"], "c", [0, 2, -1, 1], ["x", "y", "z"], ordered=ordered + ) + with opener("r") as root: + col = root["uns"]["c"] + assert spec.encoding_of(col) == (spec.CATEGORICAL, "0.2.0") + assert is_ordered(col) is ordered + assert list(read_categories(col)) == ["x", "y", "z"] + # -1 denotes missing and must render empty, not index from the end. + assert read_str_chunk(col, 0, 4) == ["x", "z", "", "y"] + + +def test_categorical_codes_are_tagged_as_arrays(new_store): + """Untagged codes make anndata fall back to its legacy reader.""" + path, opener = new_store() + with opener("a") as root: + ew.write_categorical(root["uns"], "c", [0, 1], ["x", "y"]) + with opener("r") as root: + assert spec.encoding_type(root["uns"]["c"]["codes"]) == spec.ARRAY + assert spec.encoding_type(root["uns"]["c"]["categories"]) == spec.STRING_ARRAY + + +@pytest.mark.parametrize( + "n_categories,expected", + [(2, np.int8), (200, np.int16), (40_000, np.int32)], +) +def test_categorical_code_dtype_widens_with_category_count(n_categories, expected): + from adata.elements.write import _codes_dtype + + assert _codes_dtype(n_categories) is expected + + +@pytest.mark.parametrize( + "enc,values,mask", + [ + (spec.NULLABLE_INTEGER, np.array([1, 2, 3], dtype="int32"), [False, True, False]), + (spec.NULLABLE_BOOLEAN, np.array([True, False, True]), [False, False, True]), + (spec.NULLABLE_STRING_ARRAY, ["a", "b", "c"], [True, False, False]), + ], +) +def test_masked_elements_round_trip(new_store, enc, values, mask): + path, opener = new_store() + with opener("a") as root: + ew.write_masked(root["uns"], "m", values, mask, enc) + with opener("r") as root: + col = root["uns"]["m"] + assert spec.encoding_type(col) == enc + assert element_len(col) == 3 + rendered = read_masked_column(col, 0, 3, na_repr="NA") + for i, is_missing in enumerate(mask): + assert (rendered[i] == "NA") is bool(is_missing) + + +def test_masked_write_records_na_value(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_masked( + root["uns"], "m", ["a"], [True], spec.NULLABLE_STRING_ARRAY, + na_value="NaN", + ) + with opener("r") as root: + assert spec.decode_attr(root["uns"]["m"].attrs["na-value"]) == "NaN" + + +def test_write_masked_rejects_a_non_masked_encoding(new_store): + path, opener = new_store() + with opener("a") as root: + with pytest.raises(ValueError, match="not a masked encoding"): + ew.write_masked(root["uns"], "m", [1], [False], spec.ARRAY) + + +@pytest.mark.parametrize( + "value,enc", + [ + ("hello", spec.STRING), + (42, spec.NUMERIC_SCALAR), + (3.5, spec.NUMERIC_SCALAR), + (True, spec.NUMERIC_SCALAR), + (np.float32(1.5), spec.NUMERIC_SCALAR), + ], +) +def test_scalars_are_written_as_the_right_encoding(new_store, value, enc): + path, opener = new_store() + with opener("a") as root: + ew.write_scalar(root["uns"], "s", value) + with opener("r") as root: + ds = root["uns"]["s"] + assert spec.encoding_type(ds) == enc + assert ds.shape == () + + +def test_null_round_trips(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_null(root["uns"], "n") + with opener("r") as root: + assert spec.encoding_type(root["uns"]["n"]) == spec.NULL + + +def test_sparse_shape_attr_is_written_in_the_backends_own_form(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_sparse(root["uns"], "m", [1.0], [0], [0, 1], (1, 2)) + with opener("r") as root: + group = root["uns"]["m"] + assert spec.encoding_type(group) == spec.CSR_MATRIX + assert [int(d) for d in group.attrs["shape"]] == [1, 2] + + +def test_write_sparse_rejects_a_non_sparse_encoding(new_store): + path, opener = new_store() + with opener("a") as root: + with pytest.raises(ValueError, match="not a sparse encoding"): + ew.write_sparse( + root["uns"], "m", [1.0], [0], [0, 1], (1, 2), spec.ARRAY + ) + + +def test_mappings_are_tagged_and_reused(new_store): + path, opener = new_store() + with opener("a") as root: + first = ew.write_mapping(root["uns"], "m") + ew.write_scalar(first, "x", 1) + again = ew.write_mapping(root["uns"], "m") + assert "x" in again, "an existing mapping must be reused, not cleared" + with opener("r") as root: + assert spec.encoding_of(root["uns"]["m"]) == (spec.DICT, "0.1.0") + + +def test_write_mapping_replace_clears_existing_contents(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_scalar(ew.write_mapping(root["uns"], "m"), "x", 1) + ew.write_mapping(root["uns"], "m", replace=True) + with opener("r") as root: + assert list(root["uns"]["m"].keys()) == [] + + +def test_skeleton_creates_every_optional_mapping(new_store): + path, opener = new_store() + with opener("r") as root: + for key in ("layers", "obsm", "obsp", "varm", "varp", "uns"): + assert key in root + assert spec.encoding_type(root[key]) == spec.DICT + + +# --------------------------------------------------------------------------- +# read helpers + + +def test_element_len_handles_every_column_layout(new_store): + path, opener = new_store() + with opener("a") as root: + uns = root["uns"] + ew.write_string_array(uns, "plain", ["a", "b", "c"]) + ew.write_categorical(uns, "cat", [0, 1, 0], ["x", "y"]) + ew.write_masked( + uns, "nul", ["a", "b", "c"], [False] * 3, spec.NULLABLE_STRING_ARRAY + ) + with opener("r") as root: + for key in ("plain", "cat", "nul"): + assert element_len(root["uns"][key]) == 3 + + +def test_element_len_rejects_a_scalar(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_scalar(root["uns"], "s", 1) + with opener("r") as root: + with pytest.raises(ValueError, match="scalar"): + element_len(root["uns"]["s"]) + + +def test_element_len_rejects_an_unrecognised_group(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_mapping(root["uns"], "m") + with opener("r") as root: + with pytest.raises(TypeError, match="expected 'values' or 'codes'"): + element_len(root["uns"]["m"]) + + +def test_resolve_index_prefers_the_declared_index(new_store): + """A store carrying both `_index` and a stale `obs_names` must not guess.""" + path, opener = new_store() + with opener("a") as root: + ew.write_string_array(root["obs"], "obs_names", ["wrong", "wrong", "wrong"]) + with opener("r") as root: + index, name = resolve_index(root["obs"], "obs") + assert name == "_index" + assert read_str_all(index) == ["c1", "c2", "c3"] + + +def test_resolve_index_falls_back_to_the_naming_convention(new_store): + path, opener = new_store() + with opener("a") as root: + group = ew.write_mapping(root["uns"], "frame") + ew.write_string_array(group, "obs_names", ["a", "b"]) + with opener("r") as root: + _, name = resolve_index(root["uns"]["frame"], "obs") + assert name == "obs_names" + + +def test_resolve_index_reports_what_it_looked_for(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_mapping(root["uns"], "frame") + with opener("r") as root: + with pytest.raises(KeyError, match="tried"): + resolve_index(root["uns"]["frame"], "obs") + + +def test_dataframe_columns_follows_column_order(new_store): + path, opener = new_store() + with opener("a") as root: + del root["obs"] + group = ew.write_dataframe_header( + root, "obs", ["c1", "c2"], ["zebra", "apple"] + ) + ew.write_dense(group, "apple", [1, 2]) + ew.write_dense(group, "zebra", [3, 4]) + with opener("r") as root: + # Not alphabetical, which is what the backend would otherwise give. + assert dataframe_columns(root["obs"], "_index") == ["zebra", "apple"] + + +def test_dataframe_columns_appends_undeclared_columns(new_store): + """A column on disk but missing from column-order must not vanish.""" + path, opener = new_store() + with opener("a") as root: + del root["obs"] + group = ew.write_dataframe_header(root, "obs", ["c1", "c2"], ["declared"]) + ew.write_dense(group, "declared", [1, 2]) + ew.write_dense(group, "extra", [5, 6]) + with opener("r") as root: + assert dataframe_columns(root["obs"], "_index") == ["declared", "extra"] + + +def test_read_str_chunk_rejects_an_unsupported_group(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_mapping(root["uns"], "m") + with opener("r") as root: + with pytest.raises(ValueError, match="Unsupported group encoding"): + read_str_chunk(root["uns"]["m"], 0, 1) + + +def test_read_str_chunk_renders_numbers_without_a_string_detour(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_dense(root["uns"], "n", np.array([1, 2, 3], dtype="int64")) + with opener("r") as root: + assert read_str_chunk(root["uns"]["n"], 0, 3) == ["1", "2", "3"] + + +def test_read_str_chunk_reads_a_window(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_string_array(root["uns"], "s", [f"v{i}" for i in range(10)]) + with opener("r") as root: + assert read_str_chunk(root["uns"]["s"], 3, 6) == ["v3", "v4", "v5"] + + +def test_read_str_all_spans_chunk_boundaries(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_string_array(root["uns"], "s", [f"v{i}" for i in range(25)]) + with opener("r") as root: + assert read_str_all(root["uns"]["s"], chunk_size=4) == [ + f"v{i}" for i in range(25) + ] + + +def test_read_categories_reports_a_column_with_none(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_dense(root["uns"], "plain", [1, 2]) + with opener("r") as root: + with pytest.raises(KeyError, match="Cannot find categories"): + read_categories(root["uns"]["plain"]) + + +@pytest.mark.parametrize( + "given,expected", + [ + (np.array([b"a", b"bb"]), ["a", "bb"]), + (np.array(["a", "bb"], dtype=object), ["a", "bb"]), + (np.array([1, 2]), ["1", "2"]), + (np.array([b"caf\xc3\xa9"], dtype=object), ["café"]), + ], +) +def test_decode_str_array(given, expected): + assert decode_str_array(given).tolist() == expected + + +def test_decode_str_array_preserves_shape(): + assert decode_str_array(np.array([[b"a", b"b"]])).shape == (1, 2) + + +def test_decode_str_array_replaces_undecodable_bytes(): + assert "�" in decode_str_array(np.array([b"\xff\xfe"]))[0] diff --git a/tests/test_formats.py b/tests/test_formats.py new file mode 100644 index 0000000..897f1a2 --- /dev/null +++ b/tests/test_formats.py @@ -0,0 +1,468 @@ +"""Tests for the per-format export and import paths. + +These are the surfaces that touch real file formats -- .npy, .mtx, .png, JSON +-- where a mistake produces a file that another tool rejects rather than an +exception here. +""" + +from __future__ import annotations + +import json +import sys + +import numpy as np +import pytest +from rich.console import Console + +from adata.elements import spec +from adata.elements import write as ew +from adata.formats.array import export_npy, import_npy +from adata.formats.image import export_image, import_image +from adata.formats.json_data import export_json, import_json +from adata.formats.sparse import _read_mtx, export_mtx, import_mtx +from adata.storage import open_store + +console = Console(stderr=True) + + +@pytest.fixture +def store(new_store): + path, opener = new_store() + return path, opener + + +# --------------------------------------------------------------------------- +# images + + +@pytest.fixture +def png(temp_dir): + from PIL import Image + + def _make(array, name="img.png"): + path = temp_dir / name + Image.fromarray(array).save(path) + return path + + return _make + + +@pytest.mark.parametrize( + "shape", [(4, 6), (4, 6, 3), (4, 6, 4)], ids=["gray", "rgb", "rgba"] +) +def test_image_round_trips(store, png, temp_dir, shape): + path, opener = store + array = (np.random.default_rng(0).random(shape) * 255).astype("uint8") + source = png(array) + + with opener("a") as root: + import_image(root, "uns/img", source, console) + with opener("r") as root: + assert root["uns"]["img"].shape == shape + out = temp_dir / "out.png" + export_image(root, "uns/img", out, console) + + from PIL import Image + + assert np.array_equal(np.asarray(Image.open(out)), array) + + +def test_image_export_scales_unit_range_floats(store, temp_dir): + """Floats in [0, 1] are a normalised image and scale up to 0-255.""" + path, opener = store + with opener("a") as root: + ew.write_dense(root["uns"], "img", np.array([[0.0, 0.5], [1.0, 0.25]])) + out = temp_dir / "f.png" + export_image(root, "uns/img", out, console) + + from PIL import Image + + got = np.asarray(Image.open(out)) + assert got[0, 0] == 0 and got[1, 0] == 255 + assert 120 <= got[0, 1] <= 135 + + +def test_image_export_clips_floats_already_in_byte_range(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_dense(root["uns"], "img", np.array([[-5.0, 300.0], [10.0, 20.0]])) + out = temp_dir / "f.png" + export_image(root, "uns/img", out, console) + + from PIL import Image + + got = np.asarray(Image.open(out)) + assert got[0, 0] == 0, "negatives clip to 0" + assert got[0, 1] == 255, "values above 255 clip to 255" + + +def test_image_export_handles_booleans(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_dense(root["uns"], "img", np.array([[True, False], [False, True]])) + out = temp_dir / "b.png" + export_image(root, "uns/img", out, console) + + from PIL import Image + + got = np.asarray(Image.open(out)) + assert got[0, 0] == 255 and got[0, 1] == 0 + + +def test_image_export_squeezes_a_single_channel(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_dense( + root["uns"], "img", np.zeros((3, 4, 1), dtype="uint8") + ) + out = temp_dir / "s.png" + export_image(root, "uns/img", out, console) + + from PIL import Image + + assert np.asarray(Image.open(out)).shape == (3, 4) + + +def test_image_export_rejects_a_bad_channel_count(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_dense(root["uns"], "img", np.zeros((3, 4, 2), dtype="uint8")) + with pytest.raises(ValueError, match="channels"): + export_image(root, "uns/img", temp_dir / "x.png", console) + + +def test_image_export_rejects_wrong_dimensionality(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_dense(root["uns"], "img", np.zeros((2, 2, 2, 2), dtype="uint8")) + with pytest.raises(ValueError, match="2D or 3D"): + export_image(root, "uns/img", temp_dir / "x.png", console) + + +def test_image_export_rejects_a_group(store, temp_dir): + path, opener = store + with opener("a") as root: + with pytest.raises(ValueError, match="requires a dataset"): + export_image(root, "uns", temp_dir / "x.png", console) + + +def test_image_export_rejects_an_unsupported_dtype(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_dense( + root["uns"], "img", np.array([[1 + 2j, 3 + 4j]], dtype="complex64") + ) + with pytest.raises(ValueError, match="dtype"): + export_image(root, "uns/img", temp_dir / "x.png", console) + + +def test_image_import_creates_intermediate_groups(store, png): + path, opener = store + source = png(np.zeros((2, 2, 3), dtype="uint8")) + with opener("a") as root: + import_image(root, "uns/spatial/sample/hires", source, console) + with opener("r") as root: + assert spec.encoding_type(root["uns"]["spatial"]) == spec.DICT + assert root["uns"]["spatial"]["sample"]["hires"].shape == (2, 2, 3) + + +# --------------------------------------------------------------------------- +# dense arrays + + +@pytest.mark.parametrize( + "array", + [ + np.arange(10, dtype="int32"), + np.arange(12, dtype="float32").reshape(3, 4), + np.arange(24, dtype="float64").reshape(2, 3, 4), + np.array(7.5), + ], + ids=["1d", "2d", "3d", "scalar"], +) +def test_npy_round_trips(store, temp_dir, array): + path, opener = store + source = temp_dir / "in.npy" + np.save(source, array) + + with opener("a") as root: + import_npy(root, "uns/a", source, console) + with opener("r") as root: + out = temp_dir / "out.npy" + export_npy(root, "uns/a", out, chunk_elements=5, console=console) + + assert np.array_equal(np.load(out), array) + + +def test_npy_export_streams_in_small_chunks(store, temp_dir): + """A chunk size below the array length exercises the streaming loop.""" + path, opener = store + array = np.arange(100, dtype="int64").reshape(25, 4) + with opener("a") as root: + ew.write_dense(root["uns"], "a", array) + out = temp_dir / "out.npy" + export_npy(root, "uns/a", out, chunk_elements=8, console=console) + assert np.array_equal(np.load(out), array) + + +def test_npy_export_unwraps_a_nullable_group(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_masked( + root["uns"], "m", np.array([1, 2, 3], dtype="int32"), + [False, True, False], spec.NULLABLE_INTEGER, + ) + out = temp_dir / "m.npy" + export_npy(root, "uns/m", out, chunk_elements=100, console=console) + assert np.array_equal(np.load(out), np.array([1, 2, 3], dtype="int32")) + + +def test_npy_export_rejects_a_group_it_cannot_unwrap(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_mapping(root["uns"], "m") + with pytest.raises(ValueError, match="cannot export as .npy"): + export_npy(root, "uns/m", temp_dir / "x.npy", 100, console) + + +def test_npy_export_to_stdout(store, capsysbinary): + path, opener = store + with opener("a") as root: + ew.write_dense(root["uns"], "a", np.arange(6, dtype="float32")) + export_npy(root, "uns/a", None, chunk_elements=100, console=console) + captured = capsysbinary.readouterr().out + assert captured.startswith(b"\x93NUMPY") + + +def test_npy_import_creates_intermediate_groups(store, temp_dir): + path, opener = store + source = temp_dir / "in.npy" + np.save(source, np.zeros((2, 2))) + with opener("a") as root: + import_npy(root, "uns/deep/nested/a", source, console) + with opener("r") as root: + assert root["uns"]["deep"]["nested"]["a"].shape == (2, 2) + + +# --------------------------------------------------------------------------- +# sparse matrices + + +def _write_mtx(path, header, dims, entries): + lines = [header, "% a comment"] + lines.append(" ".join(str(d) for d in dims)) + lines.extend(" ".join(str(v) for v in e) for e in entries) + path.write_text("\n".join(lines) + "\n") + return path + + +def test_mtx_round_trips(store, temp_dir): + path, opener = store + source = _write_mtx( + temp_dir / "m.mtx", + "%%MatrixMarket matrix coordinate real general", + (3, 2, 3), + [(1, 1, 1.5), (2, 2, 2.5), (3, 1, 3.5)], + ) + with opener("a") as root: + import_mtx(root, "uns/m", source, console) + with opener("r") as root: + out = temp_dir / "out.mtx" + export_mtx(root, "uns/m", out, None, 10, False, console) + + entries, dims, nnz = _read_mtx(out) + assert dims == (3, 2) + assert nnz == 3 + assert sorted(entries) == [(0, 0, 1.5), (1, 1, 2.5), (2, 0, 3.5)] + + +def test_mtx_pattern_field_defaults_values_to_one(temp_dir): + source = _write_mtx( + temp_dir / "p.mtx", + "%%MatrixMarket matrix coordinate pattern general", + (2, 2, 2), + [(1, 1), (2, 2)], + ) + entries, _, _ = _read_mtx(source) + assert [e[2] for e in entries] == [1.0, 1.0] + + +def test_mtx_rejects_a_missing_header(temp_dir): + bad = temp_dir / "bad.mtx" + bad.write_text("1 1 1\n1 1 1.0\n") + with pytest.raises(ValueError, match="MatrixMarket header"): + _read_mtx(bad) + + +def test_mtx_export_in_memory_matches_streaming(store, temp_dir): + """The --in-memory fast path must produce the same file as streaming.""" + path, opener = store + with opener("a") as root: + ew.write_sparse( + root["uns"], "m", + [1.0, 2.0, 3.0, 4.0], [0, 2, 1, 2], [0, 2, 3, 4], (3, 3), + ) + streamed = temp_dir / "s.mtx" + in_memory = temp_dir / "m.mtx" + export_mtx(root, "uns/m", streamed, None, 1, False, console) + export_mtx(root, "uns/m", in_memory, None, 1, True, console) + + a, dims_a, _ = _read_mtx(streamed) + b, dims_b, _ = _read_mtx(in_memory) + assert dims_a == dims_b + assert sorted(a) == sorted(b) + + +def test_mtx_export_head_limits_entries(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_sparse( + root["uns"], "m", + [1.0, 2.0, 3.0, 4.0], [0, 2, 1, 2], [0, 2, 3, 4], (3, 3), + ) + out = temp_dir / "h.mtx" + export_mtx(root, "uns/m", out, 2, 10, False, console) + _, _, nnz = _read_mtx(out) + assert nnz == 2 + + +def test_mtx_export_of_csc_uses_column_major_order(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_sparse( + root["uns"], "m", + [1.0, 2.0], [0, 1], [0, 1, 2], (2, 2), spec.CSC_MATRIX, + ) + out = temp_dir / "c.mtx" + export_mtx(root, "uns/m", out, None, 10, False, console) + entries, dims, _ = _read_mtx(out) + assert dims == (2, 2) + assert sorted(entries) == [(0, 0, 1.0), (1, 1, 2.0)] + + +def test_mtx_export_rejects_a_dataset(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_dense(root["uns"], "d", np.zeros((2, 2))) + with pytest.raises(ValueError, match="CSR/CSC matrix group"): + export_mtx(root, "uns/d", temp_dir / "x.mtx", None, 10, False, console) + + +def test_mtx_export_rejects_a_non_sparse_group(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_mapping(root["uns"], "m") + with pytest.raises(ValueError, match="expected 'csr_matrix'"): + export_mtx(root, "uns/m", temp_dir / "x.mtx", None, 10, False, console) + + +def test_mtx_export_detects_inconsistent_sparse_data(store, temp_dir): + """indptr claiming more nonzeros than data holds must not be exported.""" + path, opener = store + with opener("a") as root: + group = root["uns"].create_group("m") + spec.set_encoding(group, spec.CSR_MATRIX) + ew.set_shape_attr(group, (2, 2)) + ew.write_dense(group, "data", np.array([1.0])) + ew.write_dense(group, "indices", np.array([0])) + ew.write_dense(group, "indptr", np.array([0, 1, 5])) + with pytest.raises(ValueError, match="inconsistency"): + export_mtx(root, "uns/m", temp_dir / "x.mtx", None, 10, False, console) + + +def test_mtx_export_requires_a_shape_attribute(store, temp_dir): + path, opener = store + with opener("a") as root: + group = root["uns"].create_group("m") + spec.set_encoding(group, spec.CSR_MATRIX) + ew.write_dense(group, "data", np.array([1.0])) + ew.write_dense(group, "indices", np.array([0])) + ew.write_dense(group, "indptr", np.array([0, 1])) + with pytest.raises(ValueError, match="'shape' attribute"): + export_mtx(root, "uns/m", temp_dir / "x.mtx", None, 10, False, console) + + +# --------------------------------------------------------------------------- +# JSON + + +def test_json_round_trips_every_scalar_kind(store, temp_dir): + path, opener = store + source = { + "text": "hello", + "count": 42, + "rate": 0.25, + "flag": True, + "nothing": None, + "labels": ["a", "b"], + "numbers": [1, 2, 3], + "grid": [["a", "b"], ["c", "d"]], + "nested": {"deep": {"x": 1}}, + } + payload = temp_dir / "in.json" + payload.write_text(json.dumps(source)) + + with opener("a") as root: + import_json(root, "uns/t", payload, console) + with opener("r") as root: + out = temp_dir / "out.json" + export_json(root, "uns/t", out, 1000, False, console) + + assert json.loads(out.read_text()) == source + + +def test_json_ragged_list_is_kept_as_text(store, temp_dir): + """A ragged list has no array representation, so it is stored verbatim.""" + path, opener = store + payload = temp_dir / "r.json" + payload.write_text('{"ragged": [[1, 2], [3]]}') + + with opener("a") as root: + import_json(root, "uns/t", payload, console) + with opener("r") as root: + assert spec.encoding_type(root["uns"]["t"]["ragged"]) == spec.STRING + + +def test_json_export_refuses_an_oversized_array(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_dense(root["uns"], "big", np.arange(100)) + with pytest.raises(ValueError, match="max 10"): + export_json(root, "uns/big", temp_dir / "x.json", 10, False, console) + + +def test_json_export_refuses_a_sparse_matrix(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_sparse(root["uns"], "m", [1.0], [0], [0, 1], (1, 2)) + with pytest.raises(ValueError, match="Export it as .mtx"): + export_json(root, "uns/m", temp_dir / "x.json", 1000, False, console) + + +def test_json_export_can_include_attributes(store, temp_dir): + path, opener = store + with opener("a") as root: + ew.write_categorical(root["uns"], "c", [0, 1], ["x", "y"], ordered=True) + out = temp_dir / "a.json" + export_json(root, "uns/c", out, 1000, True, console) + + payload = json.loads(out.read_text()) + assert payload["__attrs__"]["encoding-type"] == "categorical" + assert payload["__attrs__"]["ordered"] is True + + +def test_json_export_to_stdout(store, capsys): + path, opener = store + with opener("a") as root: + ew.write_scalar(root["uns"], "s", "hi") + export_json(root, "uns/s", None, 1000, False, console) + assert json.loads(capsys.readouterr().out) == "hi" + + +def test_json_import_rejects_an_unrepresentable_value(store, temp_dir): + from adata.formats.json_data import _write_json_to_group + + path, opener = store + with opener("a") as root: + with pytest.raises(ValueError, match="Cannot convert"): + _write_json_to_group(root["uns"], "bad", {1, 2}) diff --git a/tests/test_invariants.py b/tests/test_invariants.py new file mode 100644 index 0000000..30720fc --- /dev/null +++ b/tests/test_invariants.py @@ -0,0 +1,493 @@ +"""Invariants that must hold regardless of the data, plus concat internals. + +These assert relationships rather than specific values -- a subset of +everything is the original, splitting partitions the rows exactly, concat +undoes split -- which catches classes of bug that example-based tests walk +past. +""" + +from __future__ import annotations + +import numpy as np +import pytest +from typer.testing import CliRunner + +from adata.cli import app +from adata.core.info import format_type_info, get_entry_type +from adata.elements import spec +from adata.elements import write as ew +from adata.storage import open_store + +ad = pytest.importorskip("anndata") +pd = pytest.importorskip("pandas") +sparse = pytest.importorskip("scipy.sparse") + +runner = CliRunner() + + +def _out(result) -> str: + return result.stdout + (result.stderr or "") + + +def _build(path, n_obs=8, n_var=5, seed=0, layout="csr"): + rng = np.random.default_rng(seed) + X = sparse.random( + n_obs, n_var, density=0.4, format="csr", dtype="float32", random_state=seed + ) + obs = pd.DataFrame( + { + "group": pd.Categorical(rng.choice(["a", "b", "c"], n_obs)), + "score": rng.normal(size=n_obs).astype("float32"), + "nullable": pd.array( + [None if i % 3 == 0 else i for i in range(n_obs)], dtype="Int32" + ), + "text": [f"t{i}" for i in range(n_obs)], + }, + index=[f"c{i}" for i in range(n_obs)], + ) + var = pd.DataFrame( + {"kind": pd.Categorical(["x"] * n_var)}, + index=[f"g{i}" for i in range(n_var)], + ) + obj = ad.AnnData(X=X if layout == "csr" else X.tocsc(), obs=obs, var=var) + obj.layers["counts"] = obj.X.copy() + obj.obsm["X_umap"] = rng.normal(size=(n_obs, 2)).astype("float32") + obj.write_h5ad(path) + return obj + + +@pytest.fixture +def source(temp_dir): + path = temp_dir / "src.h5ad" + return path, _build(path) + + +# --------------------------------------------------------------------------- +# subset invariants + + +def test_subsetting_everything_is_the_identity(source, temp_dir): + path, original = source + names = temp_dir / "all.txt" + names.write_text("\n".join(original.obs_names)) + out = temp_dir / "all.h5ad" + + assert runner.invoke( + app, ["subset", str(path), "-o", str(out), "--obs", str(names)] + ).exit_code == 0 + + got = ad.read_h5ad(out) + assert list(got.obs_names) == list(original.obs_names) + assert list(got.var_names) == list(original.var_names) + assert abs(got.X - original.X).nnz == 0 + assert got.obs["nullable"].tolist() == original.obs["nullable"].tolist() + assert np.allclose(got.obsm["X_umap"], original.obsm["X_umap"]) + + +def test_subsetting_is_idempotent(source, temp_dir): + path, original = source + keep = list(original.obs_names)[:4] + names = temp_dir / "keep.txt" + names.write_text("\n".join(keep)) + + first = temp_dir / "one.h5ad" + second = temp_dir / "two.h5ad" + assert runner.invoke( + app, ["subset", str(path), "-o", str(first), "--obs", str(names)] + ).exit_code == 0 + assert runner.invoke( + app, ["subset", str(first), "-o", str(second), "--obs", str(names)] + ).exit_code == 0 + + a, b = ad.read_h5ad(first), ad.read_h5ad(second) + assert list(a.obs_names) == list(b.obs_names) + assert abs(a.X - b.X).nnz == 0 + + +def test_subset_preserves_source_row_order_not_the_name_files(source, temp_dir): + """Selection is by membership; the store's own order is what survives.""" + path, original = source + names = temp_dir / "keep.txt" + names.write_text("c5\nc1\nc3\n") + out = temp_dir / "o.h5ad" + + assert runner.invoke( + app, ["subset", str(path), "-o", str(out), "--obs", str(names)] + ).exit_code == 0 + assert list(ad.read_h5ad(out).obs_names) == ["c1", "c3", "c5"] + + +def test_a_query_and_the_equivalent_name_list_agree(source, temp_dir): + path, original = source + expected = [n for n, g in zip(original.obs_names, original.obs["group"]) if g == "a"] + if not expected: + pytest.skip("fixture has no rows in group 'a'") + + names = temp_dir / "keep.txt" + names.write_text("\n".join(expected)) + by_name = temp_dir / "name.h5ad" + by_query = temp_dir / "query.h5ad" + + assert runner.invoke( + app, ["subset", str(path), "-o", str(by_name), "--obs", str(names)] + ).exit_code == 0 + assert runner.invoke( + app, ["subset", str(path), "-o", str(by_query), "-q", "group == a"] + ).exit_code == 0 + + a, b = ad.read_h5ad(by_name), ad.read_h5ad(by_query) + assert list(a.obs_names) == list(b.obs_names) + assert abs(a.X - b.X).nnz == 0 + + +def test_unknown_names_are_reported_and_ignored(source, temp_dir): + path, original = source + names = temp_dir / "keep.txt" + names.write_text("c0\nnot-a-cell\nc1\n") + out = temp_dir / "o.h5ad" + + result = runner.invoke( + app, ["subset", str(path), "-o", str(out), "--obs", str(names)] + ) + assert result.exit_code == 0 + assert "not found" in _out(result) + assert list(ad.read_h5ad(out).obs_names) == ["c0", "c1"] + + +# --------------------------------------------------------------------------- +# split invariants + + +def test_split_partitions_the_rows_exactly(source, temp_dir): + """Every row lands in exactly one output, and nothing is invented.""" + path, original = source + out_dir = temp_dir / "parts" + assert runner.invoke( + app, ["split", str(path), "--by", "group", "-o", str(out_dir)] + ).exit_code == 0 + + seen: list = [] + for part in sorted(out_dir.glob("*.h5ad")): + piece = ad.read_h5ad(part) + assert len(set(piece.obs["group"])) == 1, "a split must be homogeneous" + seen.extend(piece.obs_names) + + assert sorted(seen) == sorted(original.obs_names) + assert len(seen) == len(set(seen)), "no row may appear twice" + + +def test_split_then_concat_recovers_every_value(source, temp_dir): + path, original = source + out_dir = temp_dir / "parts" + assert runner.invoke( + app, ["split", str(path), "--by", "group", "-o", str(out_dir)] + ).exit_code == 0 + + parts = sorted(str(p) for p in out_dir.glob("*.h5ad")) + merged = temp_dir / "merged.h5ad" + assert runner.invoke(app, ["concat", *parts, "-o", str(merged)]).exit_code == 0 + + got = ad.read_h5ad(merged) + ref = original[list(got.obs_names)] + assert got.shape == original.shape + assert abs(got.X - ref.X).nnz == 0 + assert got.obs["nullable"].tolist() == ref.obs["nullable"].tolist() + assert list(got.obs["group"]) == list(ref.obs["group"]) + assert np.allclose(got.obsm["X_umap"], ref.obsm["X_umap"]) + + +# --------------------------------------------------------------------------- +# concat internals + + +def _two_stores(temp_dir, **kwargs): + a = temp_dir / "a.h5ad" + b = temp_dir / "b.h5ad" + return a, b + + +def test_concat_pads_a_column_missing_from_one_input(temp_dir): + """Outer join keeps the union of columns, padding where absent.""" + for name, extra in (("a", True), ("b", False)): + obs = pd.DataFrame( + {"shared": np.arange(2, dtype="int32")}, + index=[f"{name}{i}" for i in range(2)], + ) + if extra: + obs["only_a"] = np.arange(2, dtype="int32") + obs["cat_only_a"] = pd.Categorical(["p", "q"]) + obs["text_only_a"] = ["u", "v"] + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=obs, + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(temp_dir / f"{name}.h5ad") + + out = temp_dir / "m.h5ad" + result = runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), + "-o", str(out), "--join", "outer"], + ) + assert result.exit_code == 0, _out(result) + + got = ad.read_h5ad(out) + assert "only_a" in got.obs + # Absent rows become missing, not zero. + assert got.obs["only_a"].isna().tolist() == [False, False, True, True] + assert got.obs["cat_only_a"].isna().tolist() == [False, False, True, True] + assert got.obs["text_only_a"].isna().tolist()[2:] == [True, True] + + +def test_concat_inner_join_keeps_only_shared_columns(temp_dir): + for name, extra in (("a", True), ("b", False)): + obs = pd.DataFrame( + {"shared": np.arange(2, dtype="int32")}, + index=[f"{name}{i}" for i in range(2)], + ) + if extra: + obs["only_a"] = np.arange(2, dtype="int32") + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=obs, + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(temp_dir / f"{name}.h5ad") + + out = temp_dir / "m.h5ad" + assert runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), "-o", str(out)], + ).exit_code == 0 + + got = ad.read_h5ad(out) + assert list(got.obs.columns) == ["shared"] + + +def test_concat_handles_dense_matrices(temp_dir): + for name in ("a", "b"): + ad.AnnData( + X=np.arange(4, dtype="float32").reshape(2, 2), + obs=pd.DataFrame(index=[f"{name}{i}" for i in range(2)]), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(temp_dir / f"{name}.h5ad") + + out = temp_dir / "m.h5ad" + assert runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), "-o", str(out)], + ).exit_code == 0 + + got = ad.read_h5ad(out) + assert got.X.shape == (4, 2) + assert not sparse.issparse(got.X) + + +def test_concat_outer_join_fills_dense_gaps(temp_dir): + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=["a0", "a1"]), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(temp_dir / "a.h5ad") + ad.AnnData( + X=np.full((2, 2), 5.0, dtype="float32"), + obs=pd.DataFrame(index=["b0", "b1"]), + var=pd.DataFrame(index=["g2", "g3"]), + ).write_h5ad(temp_dir / "b.h5ad") + + out = temp_dir / "m.h5ad" + assert runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), + "-o", str(out), "--join", "outer", "--fill-value", "-1"], + ).exit_code == 0 + + got = ad.read_h5ad(out) + assert list(got.var_names) == ["g1", "g2", "g3"] + assert got.X[0, 2] == -1, "a cell with no value gets the fill value" + assert got.X[2, 0] == -1 + + +def test_concat_skips_obsm_absent_from_an_input(temp_dir): + for name, with_obsm in (("a", True), ("b", False)): + obj = ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=[f"{name}{i}" for i in range(2)]), + var=pd.DataFrame(index=["g1", "g2"]), + ) + if with_obsm: + obj.obsm["X_umap"] = np.zeros((2, 2), dtype="float32") + obj.write_h5ad(temp_dir / f"{name}.h5ad") + + out = temp_dir / "m.h5ad" + result = runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), "-o", str(out)], + ) + assert result.exit_code == 0, _out(result) + assert "X_umap" in _out(result) and "Skipping" in _out(result) + assert "X_umap" not in ad.read_h5ad(out).obsm + + +def test_concat_skips_obsm_whose_width_disagrees(temp_dir): + for name, width in (("a", 2), ("b", 3)): + obj = ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=[f"{name}{i}" for i in range(2)]), + var=pd.DataFrame(index=["g1", "g2"]), + ) + obj.obsm["X_umap"] = np.zeros((2, width), dtype="float32") + obj.write_h5ad(temp_dir / f"{name}.h5ad") + + out = temp_dir / "m.h5ad" + result = runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), "-o", str(out)], + ) + assert result.exit_code == 0, _out(result) + assert "disagree on shape" in _out(result) + + +def test_concat_skips_a_layer_absent_from_an_input(temp_dir): + for name, with_layer in (("a", True), ("b", False)): + obj = ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=[f"{name}{i}" for i in range(2)]), + var=pd.DataFrame(index=["g1", "g2"]), + ) + if with_layer: + obj.layers["counts"] = np.ones((2, 2), dtype="float32") + obj.write_h5ad(temp_dir / f"{name}.h5ad") + + out = temp_dir / "m.h5ad" + result = runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), "-o", str(out)], + ) + assert result.exit_code == 0, _out(result) + assert "not present in every input" in _out(result) + + +def test_concat_refuses_to_overwrite(temp_dir): + for name in ("a", "b"): + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=[f"{name}{i}" for i in range(2)]), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(temp_dir / f"{name}.h5ad") + + out = temp_dir / "m.h5ad" + out.write_text("in the way") + result = runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), "-o", str(out)], + ) + assert result.exit_code == 1 + assert "already exists" in _out(result) + + +def test_concat_rejects_an_unknown_join(temp_dir): + for name in ("a", "b"): + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame(index=[f"{name}{i}" for i in range(2)]), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(temp_dir / f"{name}.h5ad") + + result = runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), + "-o", str(temp_dir / "m.h5ad"), "--join", "sideways"], + ) + assert result.exit_code == 1 + assert "inner" in _out(result) + + +def test_concat_label_cannot_shadow_an_existing_column(temp_dir): + for name in ("a", "b"): + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame( + {"batch": pd.Categorical([name] * 2)}, + index=[f"{name}{i}" for i in range(2)], + ), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(temp_dir / f"{name}.h5ad") + + result = runner.invoke( + app, + ["concat", str(temp_dir / "a.h5ad"), str(temp_dir / "b.h5ad"), + "-o", str(temp_dir / "m.h5ad"), "--label", "batch"], + ) + assert result.exit_code == 1 + assert "collides" in _out(result) + + +# --------------------------------------------------------------------------- +# type detection + + +def test_entry_types_are_reported_for_every_encoding(new_store): + path, opener = new_store() + with opener("a") as root: + uns = root["uns"] + ew.write_dense(uns, "arr", np.arange(4)) + ew.write_dense(uns, "matrix", np.zeros((2, 2))) + ew.write_string_array(uns, "text", ["a", "b"]) + ew.write_categorical(uns, "cat", [0, 1], ["x", "y"]) + ew.write_scalar(uns, "num", 1) + ew.write_scalar(uns, "str", "s") + ew.write_null(uns, "nul") + ew.write_sparse(uns, "sp", [1.0], [0], [0, 1], (1, 2)) + ew.write_masked(uns, "msk", [1], [False], spec.NULLABLE_INTEGER) + + expected = { + "arr": "array", + "matrix": "array", + "text": "string-array", + "cat": "categorical", + "num": "scalar", + "str": "scalar", + "nul": "null", + "sp": "sparse-matrix", + "msk": "nullable-array", + } + with opener("r") as root: + for key, kind in expected.items(): + info = get_entry_type(root["uns"][key]) + assert info["type"] == kind, f"{key} reported as {info['type']}" + assert format_type_info(info).startswith("[") + + +def test_a_dict_containing_obs_names_is_not_a_dataframe(new_store): + """Structural inference must not override a declared encoding.""" + path, opener = new_store() + with opener("a") as root: + group = ew.write_mapping(root["uns"], "trap") + ew.write_string_array(group, "obs_names", ["a", "b"]) + with opener("r") as root: + assert get_entry_type(root["uns"]["trap"])["type"] == "dict" + + +def test_the_root_is_reported_as_an_anndata_object(new_store): + path, opener = new_store() + with opener("r") as root: + assert get_entry_type(root)["type"] == "anndata" + + +def test_categorical_detail_mentions_ordering(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_categorical(root["uns"], "c", [0], ["x"], ordered=True) + with opener("r") as root: + assert "ordered" in get_entry_type(root["uns"]["c"])["details"] + + +def test_masked_detail_mentions_the_na_value(new_store): + path, opener = new_store() + with opener("a") as root: + ew.write_masked( + root["uns"], "m", ["a"], [True], spec.NULLABLE_STRING_ARRAY, + na_value="NaN", + ) + with opener("r") as root: + assert "na-value=NaN" in get_entry_type(root["uns"]["m"])["details"] diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..fb57eec --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,307 @@ +"""Tests for the storage layer: backend detection, copying, and Zarr versions. + +This layer is what lets every other module stay backend-agnostic, so its job +is to make HDF5 and Zarr behave identically. The cases that matter are the +ones where they genuinely differ -- string dtypes, attribute types, codecs, +and Zarr's consolidated metadata index. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from adata.elements import spec +from adata.elements import write as ew +from adata.storage import ( + Store, + copy_attrs, + copy_dataset, + copy_path, + copy_store_contents, + copy_tree, + dataset_create_kwargs, + detect_backend, + has_valid_anndata_root_attrs, + is_dataset, + is_group, + is_zarr_path, + open_store, + zarr_format_of, +) + +zarr = pytest.importorskip("zarr") + + +# --------------------------------------------------------------------------- +# detection + + +def test_detect_backend_from_an_existing_file(temp_dir): + path = temp_dir / "x.h5ad" + with open_store(path, "w"): + pass + assert detect_backend(path) == "hdf5" + + +def test_detect_backend_from_an_existing_zarr_directory(temp_dir): + path = temp_dir / "x.zarr" + with open_store(path, "w"): + pass + assert detect_backend(path) == "zarr" + assert is_zarr_path(path) + + +@pytest.mark.parametrize( + "name,expected", [("new.zarr", "zarr"), ("new.h5ad", "hdf5"), ("new", "hdf5")] +) +def test_detect_backend_of_a_path_that_does_not_exist_yet(temp_dir, name, expected): + assert detect_backend(temp_dir / name) == expected + + +def test_detect_backend_rejects_a_plain_directory(temp_dir): + plain = temp_dir / "plain" + plain.mkdir() + with pytest.raises(ValueError, match="does not look like a Zarr store"): + detect_backend(plain) + + +def test_is_zarr_path_is_false_for_a_file(temp_dir): + f = temp_dir / "f.txt" + f.write_text("x") + assert not is_zarr_path(f) + + +# --------------------------------------------------------------------------- +# root attributes + + +def test_writable_open_stamps_the_anndata_root(new_store): + path, opener = new_store() + with opener("r") as root: + assert has_valid_anndata_root_attrs(root) + + +def test_reading_a_non_anndata_store_warns(temp_dir): + import h5py + + path = temp_dir / "plain.h5" + with h5py.File(path, "w") as f: + f.create_dataset("x", data=[1]) + + with pytest.warns(UserWarning, match="missing or invalid AnnData attrs"): + with open_store(path, "r"): + pass + + +def test_the_warning_can_be_suppressed_for_format_agnostic_commands(temp_dir): + import h5py + import warnings + + path = temp_dir / "plain.h5" + with h5py.File(path, "w") as f: + f.create_dataset("x", data=[1]) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + with open_store(path, "r", require_anndata=False): + pass + + +# --------------------------------------------------------------------------- +# zarr versions + + +@pytest.mark.parametrize("version", [2, 3]) +def test_requested_zarr_format_is_what_gets_written(temp_dir, version): + path = temp_dir / f"v{version}.zarr" + with open_store(path, "w", zarr_format=version) as store: + assert store.zarr_format == version + assert zarr_format_of(zarr.open_group(str(path))) == version + + +def test_v2_marker_files_are_written_for_v2(temp_dir): + path = temp_dir / "v2.zarr" + with open_store(path, "w", zarr_format=2): + pass + assert (path / ".zgroup").exists() + + +def test_hdf5_store_reports_no_zarr_format(temp_dir): + with open_store(temp_dir / "x.h5ad", "w") as store: + assert store.zarr_format is None + + +def test_writes_survive_a_consolidated_index(temp_dir): + """Members added to a consolidated store must be visible on reopen.""" + path = temp_dir / "c.zarr" + with open_store(path, "w") as store: + ew.write_scalar(store.root, "first", 1) + # Reopen and add another; the index from the first write is now stale. + with open_store(path, "a") as store: + ew.write_scalar(store.root, "second", 2) + + reopened = zarr.open_group(str(path)) + assert {"first", "second"} <= set(reopened.keys()) + + +# --------------------------------------------------------------------------- +# copying + + +@pytest.mark.parametrize("src_backend", ["h5ad", "zarr2", "zarr3"]) +@pytest.mark.parametrize("dst_backend", ["h5ad", "zarr2", "zarr3"]) +def test_store_contents_copy_between_every_backend_pair( + temp_dir, src_backend, dst_backend +): + """Every crossing must carry text, numbers and structure intact. + + HDF5 reports a variable-length string dataset as `object`, which Zarr + rejects, and Zarr's ` Date: Tue, 15 Sep 2026 13:34:25 +0100 Subject: [PATCH 18/23] Fix manual PyPI dispatch, the Docker examples, and split --zarr-format publish-pypi depended on check-version, which only runs for tags. GitHub skips a job whose dependency was skipped, so a manual dispatch targeting PyPI would never have published anything. Both publish jobs now use always() with the success of build asserted explicitly, accepting check-version as either passed or not applicable. The Dockerfile sets ENTRYPOINT ["adata"], so the README and index examples were running `adata adata view ...`, which Typer rejects. Dropped the repeated executable; verified against the built image. COMMANDS.md listed `--zarr-format` among split's options, but the flag did not exist -- `adata split ... --zarr-format 2` failed with an unknown option. Implemented it rather than deleting the claim: split already inherited the source store's Zarr version, so only the override was missing. Adds tests/test_docs_are_accurate.py, which extracts every `adata ...` invocation from the README and docs and checks each command and option against the real --help. That alone would not have caught this one, since the claim lived in prose rather than a fenced example, so sentences naming an option and the commands offering it are checked too. Both checks fail if the flag is removed again. Co-Authored-By: Claude Opus 5 --- .github/workflows/publish.yml | 17 ++- README.md | 2 +- docs/COMMANDS.md | 7 ++ docs/index.md | 2 +- src/adata/cli.py | 10 ++ src/adata/commands/split.py | 7 +- tests/test_commands_phase2.py | 65 ++++++++++++ tests/test_docs_are_accurate.py | 181 ++++++++++++++++++++++++++++++++ 8 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 tests/test_docs_are_accurate.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9f13d2c..2aa57e7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,7 +51,11 @@ jobs: publish-testpypi: needs: [build] - if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi' + if: | + always() + && needs.build.result == 'success' + && github.event_name == 'workflow_dispatch' + && inputs.target == 'testpypi' runs-on: ubuntu-latest environment: testpypi permissions: @@ -67,7 +71,16 @@ jobs: publish-pypi: needs: [build, check-version] - if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi') + # check-version only runs for tags, and GitHub skips a job whose + # dependency was skipped -- so without always() a manual dispatch to PyPI + # would never publish. Success of build is therefore asserted explicitly, + # and check-version is accepted as either passed or not applicable. + if: | + always() + && needs.build.result == 'success' + && (needs.check-version.result == 'success' || needs.check-version.result == 'skipped') + && (startsWith(github.ref, 'refs/tags/') + || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi')) runs-on: ubuntu-latest environment: pypi permissions: diff --git a/README.md b/README.md index 7fd0157..0570c36 100644 --- a/README.md +++ b/README.md @@ -87,5 +87,5 @@ adata concat per_sample/*.h5ad -o merged.h5ad --join outer --label sample A docker image is available on QUAY: `quay.io/cellgeni/adata-cli:latest`. Pull and run with: ```bash -docker run --rm -it -v /path/to/data:/data quay.io/cellgeni/adata-cli:latest adata view /data/your_file.h5ad +docker run --rm -it -v /path/to/data:/data quay.io/cellgeni/adata-cli:latest view /data/your_file.h5ad ``` \ No newline at end of file diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 66caad8..a5d27c6 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -233,3 +233,10 @@ New Zarr stores follow the source store's version, so a v2 input is not silently upgraded. `--zarr-format 2|3` on `create`, `subset`, `split` and `concat` overrides that. Writing a Zarr store from an `.h5ad` source defaults to v3. + +```bash +adata create out.zarr --n-obs 100 --n-var 50 --zarr-format 2 +adata subset data.zarr -o out.zarr --obs keep.txt --zarr-format 3 +adata split data.zarr --by sample -o parts/ --suffix .zarr --zarr-format 3 +adata concat a.zarr b.zarr -o merged.zarr --zarr-format 3 +``` diff --git a/docs/index.md b/docs/index.md index 1c193dd..9cd8655 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ Or run it without installing anything: ```bash docker run --rm -it -v /path/to/data:/data \ - quay.io/cellgeni/adata-cli:latest adata view /data/your_file.h5ad + quay.io/cellgeni/adata-cli:latest view /data/your_file.h5ad ``` ## Documentation diff --git a/src/adata/cli.py b/src/adata/cli.py index d3b3118..ed7dc8d 100644 --- a/src/adata/cli.py +++ b/src/adata/cli.py @@ -508,6 +508,11 @@ def split( chunk_rows: int = typer.Option( 1024, "--chunk", "-C", help="Row chunk size for dense matrices" ), + zarr_format: Optional[int] = typer.Option( + None, + "--zarr-format", + help="Zarr spec version to write (defaults to the source store's)", + ), ) -> None: """ Split a store into one file per distinct value of a column. @@ -523,6 +528,10 @@ def split( console.print("[bold red]Error:[/] --axis must be 'obs' or 'var'.") raise typer.Exit(code=1) + if zarr_format is not None and zarr_format not in (2, 3): + console.print("[bold red]Error:[/] --zarr-format must be 2 or 3.") + raise typer.Exit(code=1) + try: split_store( file=file, @@ -535,6 +544,7 @@ def split( manifest=manifest, min_size=min_size, chunk_rows=chunk_rows, + zarr_format=zarr_format, ) except Exception as e: console.print(f"[bold red]Error:[/] {e}") diff --git a/src/adata/commands/split.py b/src/adata/commands/split.py index db9ddc3..95c2409 100644 --- a/src/adata/commands/split.py +++ b/src/adata/commands/split.py @@ -60,12 +60,16 @@ def split_store( manifest: bool = True, min_size: int = 1, chunk_rows: int = 1024, + zarr_format: Optional[int] = None, ) -> List[Tuple[str, Path, int]]: """Split `file` into one store per distinct value of `column`. Returns ``(label, path, n_rows)`` for each group written. Groups smaller than `min_size` are skipped with a warning rather than producing tiny stores nobody asked for. + + `zarr_format` overrides the Zarr version of the outputs; without it they + follow the source store's. """ if axis not in ("obs", "var"): raise ValueError("--axis must be 'obs' or 'var'.") @@ -75,7 +79,8 @@ def split_store( with open_store(file, "r") as store: groups, order = group_indices(store.root, axis, column) - zarr_format = store.zarr_format + if zarr_format is None: + zarr_format = store.zarr_format if not groups: raise ValueError(f"Column {column!r} produced no groups.") diff --git a/tests/test_commands_phase2.py b/tests/test_commands_phase2.py index d7b8ec8..cf7bd8b 100644 --- a/tests/test_commands_phase2.py +++ b/tests/test_commands_phase2.py @@ -693,3 +693,68 @@ def test_import_dataframe_validates_raw_var_against_raw(tmp_path): ) assert result.exit_code == 0, _out(result) assert list(ad.read_h5ad(src).raw.var["x"]) == [1, 2, 3, 4] + + +# --------------------------------------------------------------------------- +# regressions from review of #8 + + +@pytest.mark.parametrize("version", [2, 3]) +def test_split_zarr_format_selects_the_output_version(tmp_path, version): + """COMMANDS.md advertised this before it existed.""" + src = tmp_path / "src.h5ad" + ad.AnnData( + X=np.ones((4, 2), dtype="float32"), + obs=pd.DataFrame( + {"grp": pd.Categorical(["a", "b", "a", "b"])}, + index=[f"c{i}" for i in range(4)], + ), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_h5ad(src) + + out_dir = tmp_path / "parts" + result = runner.invoke( + app, + ["split", str(src), "--by", "grp", "-o", str(out_dir), + "--suffix", ".zarr", "--zarr-format", str(version)], + ) + assert result.exit_code == 0, _out(result) + + part = out_dir / "a.zarr" + if version == 2: + assert (part / ".zgroup").exists(), "v2 writes .zgroup" + else: + assert (part / "zarr.json").exists(), "v3 writes zarr.json" + assert ad.read_zarr(part).shape == (2, 2) + + +def test_split_defaults_to_the_source_zarr_version(tmp_path): + src = tmp_path / "src.zarr" + ad.AnnData( + X=np.ones((2, 2), dtype="float32"), + obs=pd.DataFrame( + {"grp": pd.Categorical(["a", "b"])}, index=["c0", "c1"] + ), + var=pd.DataFrame(index=["g1", "g2"]), + ).write_zarr(src) + + import zarr + + source_version = zarr.open_group(str(src)).metadata.zarr_format + out_dir = tmp_path / "parts" + assert runner.invoke( + app, ["split", str(src), "--by", "grp", "-o", str(out_dir)] + ).exit_code == 0 + + part = out_dir / "a.zarr" + assert zarr.open_group(str(part)).metadata.zarr_format == source_version + + +def test_split_rejects_a_bad_zarr_format(tmp_path, sample): + result = runner.invoke( + app, + ["split", str(sample), "--by", "batch", "-o", str(tmp_path / "o"), + "--zarr-format", "9"], + ) + assert result.exit_code == 1 + assert "must be 2 or 3" in _out(result) diff --git a/tests/test_docs_are_accurate.py b/tests/test_docs_are_accurate.py new file mode 100644 index 0000000..c5b7993 --- /dev/null +++ b/tests/test_docs_are_accurate.py @@ -0,0 +1,181 @@ +"""Check that the documentation describes the CLI that actually exists. + +Every `adata ...` invocation in the README and docs is extracted and its +options checked against the real `--help` for that command. This exists +because COMMANDS.md advertised `--zarr-format` on `split` before the option +was implemented: prose drifts from code silently, and nothing else in the +suite reads the docs. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Iterator, List, Tuple + +import pytest +from typer.testing import CliRunner + +from adata.cli import app + +runner = CliRunner() + +REPO = Path(__file__).resolve().parent.parent +DOCS = [REPO / "README.md"] + sorted((REPO / "docs").glob("*.md")) + +#: Placeholders that stand in for a real path in the docs. +_PLACEHOLDER = re.compile(r"^[<{\[]|[>}\]]$") + + +def _fenced_blocks(text: str) -> Iterator[str]: + """Yield the contents of ```bash / ```shell fences.""" + for match in re.finditer(r"```(?:bash|shell|console|sh)\n(.*?)```", text, re.S): + yield match.group(1) + + +def _invocations(path: Path) -> List[Tuple[str, str]]: + """Every `adata ...` command line in a document, with its source line.""" + found: List[Tuple[str, str]] = [] + for block in _fenced_blocks(path.read_text()): + for raw in block.splitlines(): + line = raw.strip().rstrip("\\").strip() + # Strip a trailing comment and any shell redirection or pipe. + line = re.split(r"\s+#\s", line)[0].strip() + line = re.split(r"\s*[|>]\s*", line)[0].strip() + if line.startswith("adata ") and not line.startswith("adata-cli"): + found.append((line, path.name)) + return found + + +ALL_INVOCATIONS = [inv for doc in DOCS for inv in _invocations(doc)] + + +def test_the_docs_contain_examples_to_check(): + """A parsing regression here would silently make every case below vacuous.""" + assert len(ALL_INVOCATIONS) > 20, ALL_INVOCATIONS + + +def _command_path(tokens: List[str]) -> List[str]: + """The subcommand path, e.g. ['export', 'dataframe'].""" + path: List[str] = [] + for token in tokens[1:]: + if token.startswith("-"): + break + # A value rather than a subcommand name. + if path and path[0] in {"export", "import"} and len(path) == 2: + break + if not path or path[0] in {"export", "import"}: + if _looks_like_a_subcommand(token, path): + path.append(token) + continue + break + return path + + +_GROUPS = {"export", "import"} +_TOP_LEVEL = { + "view", "ls", "subset", "split", "concat", "create", "export", "import", +} +_SUBCOMMANDS = { + "export": {"dataframe", "array", "sparse", "dict", "image"}, + "import": {"dataframe", "array", "sparse", "dict", "image"}, +} + + +def _looks_like_a_subcommand(token: str, path: List[str]) -> bool: + if not path: + return token in _TOP_LEVEL + if path[0] in _GROUPS and len(path) == 1: + return token in _SUBCOMMANDS[path[0]] + return False + + +@pytest.mark.parametrize( + "line,source", + ALL_INVOCATIONS, + ids=[f"{src}:{line[:60]}" for line, src in ALL_INVOCATIONS], +) +def test_documented_invocations_use_real_commands_and_options(line, source): + tokens = line.split() + path = _command_path(tokens) + assert path, f"{source}: could not identify a command in {line!r}" + + result = runner.invoke(app, [*path, "--help"]) + assert result.exit_code == 0, f"{source}: `{' '.join(path)}` is not a command" + + help_text = result.stdout + for token in tokens[len(path) + 1 :]: + if not token.startswith("--"): + continue + option = token.split("=")[0] + if _PLACEHOLDER.search(option): + continue + assert option in help_text, ( + f"{source}: `{line}` uses {option}, which `adata " + f"{' '.join(path)} --help` does not offer" + ) + + +def test_short_options_in_the_docs_exist_too(): + """Short flags are easy to mistype and just as easy to check.""" + problems = [] + for line, source in ALL_INVOCATIONS: + tokens = line.split() + path = _command_path(tokens) + if not path: + continue + help_text = runner.invoke(app, [*path, "--help"]).stdout + for token in tokens[len(path) + 1 :]: + if not re.fullmatch(r"-[A-Za-z]", token): + continue + if token not in help_text: + problems.append(f"{source}: `{line}` uses {token}") + assert not problems, "\n".join(problems) + + +# --------------------------------------------------------------------------- +# prose claims +# +# The fenced-invocation checks above only see commands someone wrote out in +# full. The `--zarr-format` drift lived in a sentence instead -- "`--zarr-format +# 2|3` on `create`, `subset`, `split` and `concat`" -- so sentences that name an +# option and the commands offering it are checked too. + +_CLAIM = re.compile( + r"`(--[a-z][a-z0-9-]*)[^`]*`[^.]*?\bon\b((?:[^.]|\n)*?)(?:overrides|applies|works)", + re.I, +) +_COMMAND_IN_PROSE = re.compile(r"`([a-z][a-z0-9-]*)`") + + +def _prose_claims() -> List[Tuple[str, str, str]]: + """(option, command, source) for each sentence claiming support.""" + claims: List[Tuple[str, str, str]] = [] + for doc in DOCS: + for match in _CLAIM.finditer(doc.read_text()): + option = match.group(1) + for command in _COMMAND_IN_PROSE.findall(match.group(2)): + if command in _TOP_LEVEL: + claims.append((option, command, doc.name)) + return claims + + +PROSE_CLAIMS = _prose_claims() + + +def test_prose_claims_were_found(): + assert PROSE_CLAIMS, "the claim parser matched nothing; it has rotted" + + +@pytest.mark.parametrize( + "option,command,source", + PROSE_CLAIMS, + ids=[f"{src}:{cmd}{opt}" for opt, cmd, src in PROSE_CLAIMS], +) +def test_options_claimed_in_prose_exist(option, command, source): + result = runner.invoke(app, [command, "--help"]) + assert result.exit_code == 0, f"{source}: `{command}` is not a command" + assert option in result.stdout, ( + f"{source}: prose says {command} supports {option}, " + f"but `adata {command} --help` does not offer it" + ) From 05f540d33474ec5955c842e7b98707cc9e8b9544 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 13:49:55 +0100 Subject: [PATCH 19/23] Read documented options from the command tree, not from rendered help The docs test scraped `--help` output, which Rich wraps to the terminal width. It passed on a wide local terminal and failed in CI, where the width is narrower and long option names break across lines -- an assertion about rendered UI rather than about the CLI. Options now come from the Click command objects behind the Typer app, so the check is exact and width-independent. Verified passing at COLUMNS 40 through 200, and still failing at both widths when --zarr-format is removed from split again. Co-Authored-By: Claude Opus 5 --- tests/test_docs_are_accurate.py | 57 +++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/tests/test_docs_are_accurate.py b/tests/test_docs_are_accurate.py index c5b7993..d474a34 100644 --- a/tests/test_docs_are_accurate.py +++ b/tests/test_docs_are_accurate.py @@ -13,12 +13,38 @@ from pathlib import Path from typing import Iterator, List, Tuple +import click import pytest -from typer.testing import CliRunner +from typer.main import get_command from adata.cli import app -runner = CliRunner() +#: The real Click command tree. Options are read from here rather than from +#: rendered `--help` text: Rich wraps long option names at the terminal width, +#: so a help-scraping check passes on a wide terminal and fails in CI. +ROOT = get_command(app) + + +def _lookup(path: List[str]) -> click.Command: + """Resolve a subcommand path, or raise KeyError naming what is missing.""" + command: click.Command = ROOT + for name in path: + if not isinstance(command, click.Group): + raise KeyError(f"{name!r}: {path} is not a group") + found = command.get_command(click.Context(command), name) + if found is None: + raise KeyError(f"{name!r} is not a command of {path}") + command = found + return command + + +def _options(command: click.Command) -> set: + """Every option string the command accepts, long and short.""" + names = set() + for param in command.params: + names.update(param.opts) + names.update(param.secondary_opts) + return names REPO = Path(__file__).resolve().parent.parent DOCS = [REPO / "README.md"] + sorted((REPO / "docs").glob("*.md")) @@ -100,19 +126,22 @@ def test_documented_invocations_use_real_commands_and_options(line, source): path = _command_path(tokens) assert path, f"{source}: could not identify a command in {line!r}" - result = runner.invoke(app, [*path, "--help"]) - assert result.exit_code == 0, f"{source}: `{' '.join(path)}` is not a command" + try: + command = _lookup(path) + except KeyError as exc: + pytest.fail(f"{source}: `{line}` -- {exc}") - help_text = result.stdout + accepted = _options(command) for token in tokens[len(path) + 1 :]: if not token.startswith("--"): continue option = token.split("=")[0] if _PLACEHOLDER.search(option): continue - assert option in help_text, ( + assert option in accepted, ( f"{source}: `{line}` uses {option}, which `adata " - f"{' '.join(path)} --help` does not offer" + f"{' '.join(path)}` does not accept. Accepted: " + f"{', '.join(sorted(o for o in accepted if o.startswith('--')))}" ) @@ -124,11 +153,11 @@ def test_short_options_in_the_docs_exist_too(): path = _command_path(tokens) if not path: continue - help_text = runner.invoke(app, [*path, "--help"]).stdout + accepted = _options(_lookup(path)) for token in tokens[len(path) + 1 :]: if not re.fullmatch(r"-[A-Za-z]", token): continue - if token not in help_text: + if token not in accepted: problems.append(f"{source}: `{line}` uses {token}") assert not problems, "\n".join(problems) @@ -173,9 +202,11 @@ def test_prose_claims_were_found(): ids=[f"{src}:{cmd}{opt}" for opt, cmd, src in PROSE_CLAIMS], ) def test_options_claimed_in_prose_exist(option, command, source): - result = runner.invoke(app, [command, "--help"]) - assert result.exit_code == 0, f"{source}: `{command}` is not a command" - assert option in result.stdout, ( + try: + resolved = _lookup([command]) + except KeyError as exc: + pytest.fail(f"{source}: {exc}") + assert option in _options(resolved), ( f"{source}: prose says {command} supports {option}, " - f"but `adata {command} --help` does not offer it" + f"but the command does not accept it" ) From 609d579ead6d4efbd93dfbd06810474f918977d8 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 13:59:47 +0100 Subject: [PATCH 20/23] Derive the target Zarr version from the destination, and fail on unbuilt fixtures The cross-version codec fix was incomplete. _target_zarr_format defaulted an unspecified target to v3, and the subset paths never passed one, so `adata subset src.zarr -o out.zarr --zarr-format 2` on a v3 source forwarded v3-only options into a v2 array. It surfaced first through sharding rather than compressors: "Zarr format 2 arrays can only be created with shard_shape set to None". dataset_create_kwargs now takes the destination group and reads the version from it, so a call site that forgets cannot be wrong; an unknown target carries nothing version-specific rather than guessing. Fixing that exposed a second problem in the same area. Zarr requires a shard to be a whole number of chunks, so clamping a chunk to the subset size invalidates the source's shard geometry -- v3 to v3 then failed too. Chunk clamping is now one helper covering every dimension, and it drops sharding when it changes a chunk, since sharding is storage layout rather than data. The dense path had its own inline clamp that missed this; it now shares the helper. Separately, the compatibility suite turned any failed fixture build into a skip, so a broken release pin could have left that CI job green having checked nothing. A build failure now fails wherever the fixtures are required -- CI, or ADATA_REQUIRE_VERSION_FIXTURES=1 -- and stays a skip locally, where a broken environment should not block unrelated work. The job also asserts that a plausible number of cases were collected, so a marker or collection mistake cannot pass as success either. Co-Authored-By: Claude Opus 5 --- .github/workflows/tests.yml | 19 +++++++- src/adata/core/subset.py | 43 ++++++++++++----- src/adata/storage/__init__.py | 26 ++++++++-- tests/reference_stores.py | 14 ++++++ tests/test_anndata_versions.py | 14 +++++- tests/test_storage.py | 88 ++++++++++++++++++++++++++++++++++ 6 files changed, 184 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4c5e22f..e69a58e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -103,4 +103,21 @@ jobs: run: uv sync --extra dev --frozen - name: Run compatibility tests - run: uv run pytest -v -W default tests/test_anndata_versions.py -m integration + env: + # A store that will not build must fail this job, not skip it -- + # otherwise a broken pin leaves it green having checked nothing. + ADATA_REQUIRE_VERSION_FIXTURES: "1" + run: | + uv run pytest -v -W default tests/test_anndata_versions.py \ + -m integration + + - name: Confirm the compatibility cases actually ran + env: + ADATA_REQUIRE_VERSION_FIXTURES: "1" + run: | + # Belt and braces: assert a non-trivial number of cases passed, so a + # collection or marker mistake cannot pass as success either. + count=$(uv run pytest -q tests/test_anndata_versions.py -m integration \ + --collect-only 2>/dev/null | grep -c "::" || true) + echo "collected $count compatibility cases" + test "$count" -ge 100 diff --git a/src/adata/core/subset.py b/src/adata/core/subset.py index 3cbbd50..a61089a 100644 --- a/src/adata/core/subset.py +++ b/src/adata/core/subset.py @@ -134,7 +134,9 @@ def _copy_rows( return copy_tree(src_ds, dst_parent, name) target_backend = _target_backend(dst_parent) - kw = dataset_create_kwargs(src_ds, target_backend=target_backend) + kw = dataset_create_kwargs( + src_ds, target_backend=target_backend, dst_parent=dst_parent + ) kw = _clamp_chunks(kw, len(indices)) ds = create_dataset( dst_parent, @@ -146,17 +148,32 @@ def _copy_rows( return ds -def _clamp_chunks(kw: dict, n_rows: int) -> dict: - """Shrink a forwarded chunk shape to fit the subset. +def _clamp_chunks(kw: dict, *out_shape: int) -> dict: + """Shrink a forwarded chunk shape to fit the subset's dimensions. + + h5py rejects a chunk larger than the dataset, so a chunked source subset + below its own chunk size would otherwise fail outright. - h5py rejects a chunk larger than the dataset, so a chunked source column - subset below its own chunk size would otherwise fail outright. + Zarr additionally requires a shard to be a whole number of chunks, so a + clamped chunk invalidates the source's shard geometry. Sharding is a + storage-layout choice rather than data, so it is dropped and left to the + backend rather than recomputed into something arbitrary. """ chunks = kw.get("chunks") - if isinstance(chunks, (tuple, list)) and len(chunks) >= 1 and n_rows > 0: - clamped = (min(int(chunks[0]), n_rows),) + tuple(int(c) for c in chunks[1:]) - kw = dict(kw) - kw["chunks"] = clamped + if not isinstance(chunks, (tuple, list)) or not chunks: + return kw + + original = tuple(int(c) for c in chunks) + clamped = tuple( + min(c, out_shape[i]) if i < len(out_shape) and out_shape[i] > 0 else c + for i, c in enumerate(original) + ) + if clamped == original: + return kw + + kw = dict(kw) + kw["chunks"] = clamped + kw.pop("shards", None) return kw @@ -224,10 +241,10 @@ def subset_dense_matrix( out_var = len(var_idx) if var_idx is not None else n_var target_backend = _target_backend(dst_parent) - kw = dataset_create_kwargs(src, target_backend=target_backend) - chunks = kw.get("chunks") - if isinstance(chunks, (tuple, list)) and len(chunks) >= 2: - kw["chunks"] = (min(int(chunks[0]), out_obs), min(int(chunks[1]), out_var)) + kw = dataset_create_kwargs( + src, target_backend=target_backend, dst_parent=dst_parent + ) + kw = _clamp_chunks(kw, out_obs, out_var) dst = create_dataset( dst_parent, diff --git a/src/adata/storage/__init__.py b/src/adata/storage/__init__.py index e446202..96d351d 100644 --- a/src/adata/storage/__init__.py +++ b/src/adata/storage/__init__.py @@ -234,7 +234,11 @@ def copy_attrs(src_attrs: Any, dst_attrs: Any, *, target_backend: str) -> None: def dataset_create_kwargs( - src: Any, *, target_backend: str, zarr_format: Optional[int] = None + src: Any, + *, + target_backend: str, + zarr_format: Optional[int] = None, + dst_parent: Any = None, ) -> dict: """Derive creation kwargs that carry a source's layout onto a new dataset. @@ -242,7 +246,11 @@ def dataset_create_kwargs( can express them; codecs that do not survive the crossing are dropped rather than forwarded into an error. """ + # Prefer the destination's own version over anything the caller guessed, + # so a call site that forgets to pass one still behaves correctly. kw_target = zarr_format + if dst_parent is not None: + kw_target = zarr_format_of(dst_parent) kw: dict = {} chunks = getattr(src, "chunks", None) if chunks is not None: @@ -259,7 +267,11 @@ def dataset_create_kwargs( kw["fillvalue"] = src.fillvalue if target_backend == "zarr" and is_zarr_array(src): src_zarr_format = getattr(getattr(src, "metadata", None), "zarr_format", None) - same_version = src_zarr_format == _target_zarr_format(kw_target) + target_format = _target_zarr_format(kw_target) + # Only when both versions are known and equal can codecs travel. + same_version = ( + target_format is not None and src_zarr_format == target_format + ) # Codecs only travel between stores of the same Zarr version: v2 holds # numcodecs objects, v3 holds its own codec classes, and neither @@ -294,7 +306,7 @@ def dataset_create_kwargs( shards = getattr(src, "shards", None) except Exception: shards = None - if shards is not None and _target_zarr_format(kw_target) == 3: + if shards is not None and target_format == 3: kw["shards"] = shards try: @@ -394,7 +406,13 @@ def create_dataset( def _target_zarr_format(zarr_format: Optional[int]) -> Optional[int]: - return zarr_format if zarr_format is not None else 3 + """The destination's Zarr version, or None when the caller did not say. + + Deliberately not defaulting to 3: guessing meant v3-only options such as + sharding were forwarded into v2 arrays, which reject them outright. + Unknown means "carry nothing version-specific". + """ + return zarr_format def _is_string_src(src: Any) -> bool: diff --git a/tests/reference_stores.py b/tests/reference_stores.py index d70925b..c7b0b2b 100644 --- a/tests/reference_stores.py +++ b/tests/reference_stores.py @@ -72,6 +72,20 @@ def offline() -> bool: return os.environ.get("ADATA_SKIP_VERSION_FIXTURES", "").strip() not in ("", "0") +def must_build() -> bool: + """Whether a store that fails to build should fail the test. + + Locally a broken environment is a nuisance and skipping is reasonable. In + CI it is the whole point of the job: turning a failed build into a skip + would let a bad release pin, or a fixture script that no longer runs + anywhere, leave the job green having checked nothing. + """ + override = os.environ.get("ADATA_REQUIRE_VERSION_FIXTURES", "").strip() + if override not in ("", "0"): + return True + return os.environ.get("CI", "").strip().lower() in ("1", "true") + + def build(release: Release, fmt: str, out_dir: Path) -> Path: """Write one reference store, returning its path.""" out_dir.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_anndata_versions.py b/tests/test_anndata_versions.py index 4bf5e8a..5c5a06f 100644 --- a/tests/test_anndata_versions.py +++ b/tests/test_anndata_versions.py @@ -32,6 +32,7 @@ RELEASES, ReferenceCache, ReferenceUnavailable, + must_build, offline, uv_available, ) @@ -66,11 +67,20 @@ def fmt(request) -> str: @pytest.fixture def store(cache, release, fmt) -> Path: - """A reference store, or a skip explaining why it could not be built.""" + """A reference store for this release and format. + + A build failure fails the test wherever these fixtures are required -- CI, + or ADATA_REQUIRE_VERSION_FIXTURES=1 -- so a broken pin cannot quietly + reduce the whole job to skips. Elsewhere it degrades to a skip, since a + local environment problem should not block unrelated work. + """ try: return cache.get(release, fmt) except ReferenceUnavailable as exc: - pytest.skip(f"could not build {release.label} ({fmt}): {exc}") + message = f"could not build {release.label} ({fmt}): {exc}" + if must_build(): + pytest.fail(message) + pytest.skip(message) def _out(result) -> str: diff --git a/tests/test_storage.py b/tests/test_storage.py index fb57eec..926627f 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -305,3 +305,91 @@ def test_store_close_is_safe_to_call_twice(temp_dir): store = open_store(temp_dir / "x.h5ad", "w") store.close() store.close() + + +# --------------------------------------------------------------------------- +# regressions from review of #9 + + +def test_target_format_is_taken_from_the_destination(temp_dir): + """Callers that forget to pass a format must still get correct kwargs. + + Defaulting an unknown target to v3 meant v3-only options -- sharding + especially -- were forwarded into v2 arrays, which reject them. + """ + src_path = temp_dir / "v3.zarr" + with open_store(src_path, "w", zarr_format=3) as store: + ew.write_dense(store.root, "a", np.arange(64).reshape(8, 8)) + + dst_path = temp_dir / "v2.zarr" + with open_store(src_path, "r") as src, open_store( + dst_path, "w", zarr_format=2 + ) as dst: + kwargs = dataset_create_kwargs( + src.root["a"], target_backend="zarr", dst_parent=dst.root + ) + assert "shards" not in kwargs + assert "compressors" not in kwargs and "compressor" not in kwargs + + # With no destination and no format, nothing version-specific travels. + blind = dataset_create_kwargs(src.root["a"], target_backend="zarr") + assert "shards" not in blind + + +@pytest.mark.parametrize("target", [2, 3]) +def test_subsetting_a_sharded_v3_store(temp_dir, target): + """A shard must survive or be dropped, never break array creation. + + Zarr requires a shard to be a whole number of chunks, so clamping the + chunk to the subset size invalidates the source's shard geometry. + """ + src_path = temp_dir / "src.zarr" + with open_store(src_path, "w", zarr_format=3) as store: + root = store.root + ew.write_dataframe_header(root, "obs", [f"c{i}" for i in range(8)], []) + ew.write_dataframe_header(root, "var", [f"g{i}" for i in range(4)], []) + ew.ensure_anndata_skeleton(root) + from adata.storage import create_dataset + + matrix = create_dataset( + root, "X", shape=(8, 4), dtype="float32", + chunks=(4, 2), shards=(8, 4), + ) + matrix[...] = np.arange(32, dtype="float32").reshape(8, 4) + spec.set_encoding(matrix, spec.ARRAY) + + names = temp_dir / "keep.txt" + names.write_text("c0\nc2\nc4\n") + out = temp_dir / f"out{target}.zarr" + + from adata.core.subset import subset_h5ad + from rich.console import Console + + subset_h5ad( + file=src_path, output=out, obs_file=names, var_file=None, + console=Console(stderr=True), zarr_format=target, + ) + + with open_store(out, "r") as store: + assert store.zarr_format == target + assert store.root["X"].shape == (3, 4) + expected = np.arange(32, dtype="float32").reshape(8, 4)[[0, 2, 4]] + assert np.array_equal(store.root["X"][...], expected) + + +def test_clamping_a_chunk_drops_an_incompatible_shard(): + from adata.core.subset import _clamp_chunks + + kept = _clamp_chunks({"chunks": (4, 2), "shards": (8, 4)}, 100, 100) + assert kept["shards"] == (8, 4), "an unclamped chunk keeps its shard" + + dropped = _clamp_chunks({"chunks": (4, 2), "shards": (8, 4)}, 3, 4) + assert dropped["chunks"] == (3, 2) + assert "shards" not in dropped + + +def test_clamping_handles_one_dimensional_chunks(): + from adata.core.subset import _clamp_chunks + + assert _clamp_chunks({"chunks": (100,)}, 5)["chunks"] == (5,) + assert _clamp_chunks({}, 5) == {} From ef1cfbbfc09829e9b734e36392f55b5814b4b2dd Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 14:05:25 +0100 Subject: [PATCH 21/23] Count compatibility cases from the run, not from --collect-only The guard added in the previous commit counted lines containing "::" in `pytest -q --collect-only` output, which prints "" instead -- so it always found zero and failed the job it was meant to protect. That is the same mistake as the help-scraping docs test: asserting on formatted output rather than on data. The check now parses the JUnit XML the run already produces and requires at least 150 executed cases, which is what "the job actually ran something" means. Verified both ways: 204 ran against a real run, and 0 when every case is skipped via ADATA_SKIP_VERSION_FIXTURES. Co-Authored-By: Claude Opus 5 --- .github/workflows/tests.yml | 24 +++++++++++++++--------- .gitignore | 1 + 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e69a58e..9cbb1bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,15 +109,21 @@ jobs: ADATA_REQUIRE_VERSION_FIXTURES: "1" run: | uv run pytest -v -W default tests/test_anndata_versions.py \ - -m integration + -m integration --junitxml=compat-results.xml - name: Confirm the compatibility cases actually ran - env: - ADATA_REQUIRE_VERSION_FIXTURES: "1" + if: always() run: | - # Belt and braces: assert a non-trivial number of cases passed, so a - # collection or marker mistake cannot pass as success either. - count=$(uv run pytest -q tests/test_anndata_versions.py -m integration \ - --collect-only 2>/dev/null | grep -c "::" || true) - echo "collected $count compatibility cases" - test "$count" -ge 100 + # Belt and braces: a marker typo or a collection mistake would make + # the step above pass while executing nothing at all. + python - <<'EOF' + import sys, xml.etree.ElementTree as ET + root = ET.parse("compat-results.xml").getroot() + suite = root.find("testsuite") if root.tag == "testsuites" else root + total = int(suite.get("tests", 0)) + skipped = int(suite.get("skipped", 0)) + ran = total - skipped + print(f"{ran} compatibility cases ran ({skipped} skipped of {total})") + if ran < 150: + sys.exit(f"expected at least 150 cases to run, got {ran}") + EOF diff --git a/.gitignore b/.gitignore index f06a69c..fb49bae 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ coverage.xml htmlcov/ .pytest_cache/ pytest-results*.xml +compat-results.xml From f9894c60562764153e78ce4b403021b270d38af0 Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 14:29:08 +0100 Subject: [PATCH 22/23] Publish as pyadata-cli; adata-cli was taken on PyPI The short name was free when this was planned but has since been claimed by an unrelated "AData CLI" project, uploaded 2026-04-04. Publishing would have failed against a project we do not own, so the distribution is now `pyadata-cli`, matching the pending publisher configured on PyPI. Only the distribution name changes. The repository, the import package, the command and the documentation all stay `adata` / `adata-cli`, so the sole user-visible difference is `pip install pyadata-cli`. That mismatch is explained where the install is documented rather than left to surprise anyone. __version__ reads its metadata under the new name, so `adata --version` keeps working; verified by installing the built wheel into a clean environment. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 3 +- README.md | 5 +++- docs/index.md | 5 +++- pyproject.toml | 5 +++- src/adata/__init__.py | 2 +- uv.lock | 68 +++++++++++++++++++++---------------------- 6 files changed, 49 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1732dfb..1bd7f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ AnnData files, and added four commands. ### Renamed -- Distribution `adata-cli`, import package `adata`, command `adata`. +- Distribution `pyadata-cli` on PyPI (the short name was already taken by + an unrelated project), import package `adata`, command `adata`. - `info` is now `view`. - The `h5ad` command and the `info` subcommand remain as aliases that warn and then run normally. **Both are removed in 1.0.0.** diff --git a/README.md b/README.md index 0570c36..06258e0 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,12 @@ A command-line tool for exploring huge AnnData stores (`.h5ad` and `.zarr`) with ## Installation ```bash -pip install adata-cli +pip install pyadata-cli ``` +The command is `adata`. The distribution is named `pyadata-cli` because +`adata-cli` was already taken on PyPI by an unrelated project. + From source with [uv](https://docs.astral.sh/uv/): ```bash git clone https://github.com/cellgeni/adata-cli.git diff --git a/docs/index.md b/docs/index.md index 9cd8655..044bf18 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,9 +12,12 @@ overkill. ## Install ```bash -pip install adata-cli +pip install pyadata-cli ``` +The command is `adata`. The distribution is named `pyadata-cli` because +`adata-cli` was already taken on PyPI by an unrelated project. + Or run it without installing anything: ```bash diff --git a/pyproject.toml b/pyproject.toml index 7e7ff86..631400c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [project] -name = "adata-cli" +# The short name `adata-cli` was taken on PyPI by an unrelated project, +# so the distribution is published as `pyadata-cli`. The import package +# and the command are both still `adata`. +name = "pyadata-cli" version = "0.5.0" description = "Streaming CLI for exploring and editing large AnnData .h5ad and .zarr stores" readme = "README.md" diff --git a/src/adata/__init__.py b/src/adata/__init__.py index 770b20d..56384b5 100644 --- a/src/adata/__init__.py +++ b/src/adata/__init__.py @@ -3,7 +3,7 @@ from importlib.metadata import PackageNotFoundError, version as _version try: - __version__ = _version("adata-cli") + __version__ = _version("pyadata-cli") except PackageNotFoundError: # pragma: no cover - running from a source tree __version__ = "0.0.0+unknown" diff --git a/uv.lock b/uv.lock index 256d4fb..f37aef1 100644 --- a/uv.lock +++ b/uv.lock @@ -10,40 +10,6 @@ resolution-markers = [ "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] -[[package]] -name = "adata-cli" -version = "0.5.0" -source = { editable = "." } -dependencies = [ - { name = "h5py" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "rich" }, - { name = "typer" }, - { name = "zarr" }, -] - -[package.optional-dependencies] -dev = [ - { name = "anndata" }, - { name = "pytest" }, - { name = "pytest-cov" }, -] - -[package.metadata] -requires-dist = [ - { name = "anndata", marker = "extra == 'dev'", specifier = ">=0.13" }, - { name = "h5py", specifier = ">=3.15.1" }, - { name = "numpy", specifier = ">=2.3.5" }, - { name = "pillow", specifier = ">=12.1.0" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.4" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.0.0" }, - { name = "rich", specifier = ">=14.2.0" }, - { name = "typer", specifier = ">=0.20.0" }, - { name = "zarr", specifier = ">=3.1.5" }, -] -provides-extras = ["dev"] - [[package]] name = "anndata" version = "0.13.3.post0" @@ -518,6 +484,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pyadata-cli" +version = "0.5.0" +source = { editable = "." } +dependencies = [ + { name = "h5py" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "rich" }, + { name = "typer" }, + { name = "zarr" }, +] + +[package.optional-dependencies] +dev = [ + { name = "anndata" }, + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "anndata", marker = "extra == 'dev'", specifier = ">=0.13" }, + { name = "h5py", specifier = ">=3.15.1" }, + { name = "numpy", specifier = ">=2.3.5" }, + { name = "pillow", specifier = ">=12.1.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.0.0" }, + { name = "rich", specifier = ">=14.2.0" }, + { name = "typer", specifier = ">=0.20.0" }, + { name = "zarr", specifier = ">=3.1.5" }, +] +provides-extras = ["dev"] + [[package]] name = "pydantic" version = "2.13.5" From 338893e7a093c328f7d466f9bd4cbc7038c9d60b Mon Sep 17 00:00:00 2001 From: Aljes Date: Tue, 15 Sep 2026 14:54:50 +0100 Subject: [PATCH 23/23] Stop nesting Rich live displays, and fail fast on a hang A CI run hung for the full 20-minute job timeout and was cancelled, reporting only that an orphan pytest process had been killed. The last test to complete was the one before the new split --zarr-format cases; py3.13 passed the same commit, and it does not reproduce locally, so it is timing-dependent. subset_h5ad wrapped its whole body in `console.status(...)` and then opened a `Progress` on the same console inside it. Rich allows one live display per console, and `split` repeats that pattern once per output group. The spinner is now stopped explicitly once the selection is known, before the progress bar starts, with a finally to guarantee it stops on any path. Whether or not that was the cause, a hang should not cost twenty minutes and leave no evidence. pytest-timeout is now a dev dependency with a 300s suite-wide limit, so a stuck test fails with a traceback pointing at it; the compatibility tests get 1800s, since assembling six environments on a cold uv cache is legitimately slow. Verified the timeout fires. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 3 +++ pytest.ini | 2 +- src/adata/core/subset.py | 11 ++++++++++- tests/test_anndata_versions.py | 4 +++- uv.lock | 14 ++++++++++++++ 5 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 631400c..1d1ebaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,9 @@ Changelog = "https://github.com/cellgeni/adata-cli/blob/main/CHANGELOG.md" dev = [ "pytest>=8.3.4", "pytest-cov>=6.0.0", + # Turns a hang into a failure with a traceback. Without it a stuck test + # burns the CI job's whole timeout and reports nothing useful. + "pytest-timeout>=2.3.0", # Used only by tests/test_anndata_roundtrip.py, to check that what this # tool reads and writes matches what anndata itself produces. "anndata>=0.13", diff --git a/pytest.ini b/pytest.ini index 45b5a3c..e7b2969 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,7 +3,7 @@ testpaths = tests python_files = test_*.py python_classes = Test* python_functions = test_* -addopts = -v --strict-markers --tb=short +addopts = -v --strict-markers --tb=short --timeout=300 --timeout-method=thread markers = slow: marks tests as slow (deselect with '-m "not slow"') integration: builds environments with uv and needs the network on first run diff --git a/src/adata/core/subset.py b/src/adata/core/subset.py index a61089a..09e7767 100644 --- a/src/adata/core/subset.py +++ b/src/adata/core/subset.py @@ -571,7 +571,12 @@ def subset_h5ad( with open_store(file, "r") as probe: zarr_format = probe.zarr_format - with console.status("[magenta]Opening files...[/]"): + # Rich allows only one live display per console, and the progress bar + # below is a second one. The spinner is therefore stopped explicitly once + # the selection is known, rather than left running around it. + status = console.status("[magenta]Opening files...[/]") + status.start() + try: with open_store(file, "r") as src_store, open_store( dst_path, "w", zarr_format=zarr_format ) as dst_store: @@ -595,6 +600,8 @@ def subset_h5ad( if var_idx is not None and "var" in src: var_keep = set(read_names(src, "var", var_idx)) + status.stop() + tasks: List[str] = [] if "obs" in src: tasks.append("obs") @@ -745,6 +752,8 @@ def subset_h5ad( ) _ensure_optional_anndata_groups(dst) + finally: + status.stop() if inplace: if file.exists(): diff --git a/tests/test_anndata_versions.py b/tests/test_anndata_versions.py index 5c5a06f..b8ded8f 100644 --- a/tests/test_anndata_versions.py +++ b/tests/test_anndata_versions.py @@ -37,7 +37,9 @@ uv_available, ) -pytestmark = pytest.mark.integration +# Building six environments on a cold uv cache is slow, so these get much +# longer than the suite-wide limit. +pytestmark = [pytest.mark.integration, pytest.mark.timeout(1800)] runner = CliRunner() diff --git a/uv.lock b/uv.lock index f37aef1..adb6d8d 100644 --- a/uv.lock +++ b/uv.lock @@ -502,6 +502,7 @@ dev = [ { name = "anndata" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "pytest-timeout" }, ] [package.metadata] @@ -512,6 +513,7 @@ requires-dist = [ { name = "pillow", specifier = ">=12.1.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.4" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.0.0" }, + { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.3.0" }, { name = "rich", specifier = ">=14.2.0" }, { name = "typer", specifier = ">=0.20.0" }, { name = "zarr", specifier = ">=3.1.5" }, @@ -661,6 +663,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, ] +[[package]] +name = "pytest-timeout" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"