diff --git a/docs/combine-endpoint.md b/docs/combine-endpoint.md index 2fa234e..841f75f 100644 --- a/docs/combine-endpoint.md +++ b/docs/combine-endpoint.md @@ -533,7 +533,7 @@ a `NOT` keeps members it should have removed. The warning names the operand and **2. Two sides that can never intersect.** VFB IDs come in namespaces: `FBbt_` is an anatomy class (a cell type or region), `VFB_` is an individual (one imaged or reconstructed specimen), `VFBexp_` is -an expression pattern, `FBlc_` a dataset or transcriptomic cluster, `FBgn_` a gene. An `AND` between +an expression pattern, `FBlc…` / `VFBseq_` a dataset or transcriptomic cluster, `FBgn_` a gene. An `AND` between two different namespaces is empty *by construction*, not by biology. The endpoint diagnoses this on the step that went empty and names the kinds in words ("individual on the left, anatomy class on the right"), because the user who made the mistake is precisely the one who does not yet read ID prefixes diff --git a/docs/http-api.md b/docs/http-api.md index 75853cd..4320f99 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -61,7 +61,7 @@ GET /get_term_info?id=FBbt_00007401 | Parameter | | |---|---| -| `id` | **Required.** A VFB short_form: `FBbt_…` (anatomy class), `VFB_…` (individual), `VFBexp_…`, `FBgn_…`, `FBlc_…`. | +| `id` | **Required.** A VFB short_form: `FBbt_…` (anatomy class), `VFB_…` (individual), `VFBexp_…`, `FBgn_…`, `FBlc…` / `VFBseq_…` (scRNAseq datasets and clusters). | | `force_refresh` | `true` bypasses the result cache for this call. | ### Query previews may be pending diff --git a/src/test/scrnaseq_fixtures.py b/src/test/scrnaseq_fixtures.py new file mode 100644 index 0000000..2f72f1c --- /dev/null +++ b/src/test/scrnaseq_fixtures.py @@ -0,0 +1,69 @@ +"""Live fixtures for the scRNAseq (transcriptomics) tests. + +The scRNAseq data in VFB is moving from the old FlyBase/SCEA loads (Datasets, +Clusters, Samples and Assays with FlyBase ``FBlc...`` ids) to the +vfb-scRNAseq repo's loads (``VFBseq_...`` ids). The two never coexist for one +study, so a hard-coded ``FBlc`` fixture disappears when its study is switched. + +Each helper keeps the preferred (known-good) id while the database still has +data for it, and otherwise picks the first term that has the data the test +needs, so the tests run unchanged before, during and after the switch. +""" + +from vfbquery.vfb_queries import vc, get_dict_cursor + +#: Id prefixes of scRNAseq Datasets / Clusters: old FlyBase loads, new VFBseq loads. +SCRNASEQ_ID_PREFIXES = ("FBlc", "VFBseq_") + + +def _first(query): + rows = get_dict_cursor()(vc.nc.commit_list([query])) + return rows[0]["id"] if rows else None + + +def _pick(preferred, has_data, any_with_data): + """``preferred`` if ``has_data`` (a query on ``$id``) finds it, else the + first id ``any_with_data`` returns, else ``preferred`` (so the test fails + on its own empty-result assertion rather than here).""" + found = _first(has_data.replace("$id", "'%s'" % preferred)) + return found or _first(any_with_data) or preferred + + +def cluster_with_expression(preferred): + """A Cluster with expressed genes and a cell type.""" + match = ("MATCH (c:Individual:Cluster)-[:expresses]->(:Gene:Class) " + "MATCH (c)-[:composed_primarily_of]->(:Anatomy) ") + return _pick( + preferred, + match + "WHERE c.short_form = $id RETURN c.short_form AS id LIMIT 1", + match + "RETURN DISTINCT c.short_form AS id ORDER BY id LIMIT 1") + + +def dataset_with_clusters(preferred): + """An scRNAseq DataSet with Clusters that have a cell type.""" + match = ("MATCH (:Anatomy)<-[:composed_primarily_of]-(:Individual:Cluster)" + "-[:has_source]->(ds:scRNAseq_DataSet) ") + return _pick( + preferred, + match + "WHERE ds.short_form = $id RETURN ds.short_form AS id LIMIT 1", + match + "RETURN DISTINCT ds.short_form AS id ORDER BY id LIMIT 1") + + +def gene_with_clusters(preferred): + """A Gene expressed in Clusters that have a cell type.""" + match = ("MATCH (:Anatomy)<-[:composed_primarily_of]-(:Individual:Cluster)" + "-[:expresses]->(g:Gene:Class) ") + return _pick( + preferred, + match + "WHERE g.short_form = $id RETURN g.short_form AS id LIMIT 1", + match + "RETURN g.short_form AS id LIMIT 1") + + +def anatomy_with_clusters(preferred): + """A cell type that scRNAseq Clusters are composed primarily of.""" + match = ("MATCH (a:Class:Anatomy)<-[:composed_primarily_of]-(:Cluster)" + "-[:has_source]->(:scRNAseq_DataSet) ") + return _pick( + preferred, + match + "WHERE a.short_form = $id RETURN a.short_form AS id LIMIT 1", + match + "RETURN a.short_form AS id LIMIT 1") diff --git a/src/test/test_query_performance.py b/src/test/test_query_performance.py index 3a72021..1e2ddce 100644 --- a/src/test/test_query_performance.py +++ b/src/test/test_query_performance.py @@ -43,6 +43,7 @@ get_scrnaseq_dataset_data, ) from vfbquery.vfb_connectivity import query_connectivity +from .scrnaseq_fixtures import cluster_with_expression, dataset_with_clusters class QueryPerformanceTest(unittest.TestCase): @@ -552,11 +553,13 @@ def test_11_transcriptomics_queries(self): # clusterExpression - these FBlc/FBgn ids are real and return data # (4588 / 9 / 13); no try/except-Skipped swallow, so a real failure or an - # empty result now fails (a backend outage is skipped upstream). + # empty result now fails (a backend outage is skipped upstream). The FBlc + # ids go when their study moves to a VFBseq load; a live one with data is + # used then (scrnaseq_fixtures). result, duration, success = self._time_query( "clusterExpression (example cluster)", get_cluster_expression, - "FBlc0005370", # cluster with expression data + cluster_with_expression("FBlc0005370"), # cluster with expression data return_dataframe=False, limit=10 ) @@ -582,7 +585,7 @@ def test_11_transcriptomics_queries(self): result, duration, success = self._time_query( "scRNAdatasetData (example dataset)", get_scrnaseq_dataset_data, - "FBlc0005362", # scRNAseq dataset with clusters + dataset_with_clusters("FBlc0005362"), # scRNAseq dataset with clusters return_dataframe=False, limit=10 ) diff --git a/src/test/test_term_info_fallback.py b/src/test/test_term_info_fallback.py index 4e4d1f1..bd56c89 100644 --- a/src/test/test_term_info_fallback.py +++ b/src/test/test_term_info_fallback.py @@ -60,7 +60,9 @@ def test_exclusions_are_scoped_to_the_anatomical_branch(): indexer's parameter query excludes them; the term_info index as a whole does not. Applying the list globally would refuse every Cluster.""" assert "FBlc" in tif.ANATOMICAL_EXCLUDED_ID_PREFIXES + assert "VFBseq_" in tif.ANATOMICAL_EXCLUDED_ID_PREFIXES assert tif.choose_indexer(["Individual", "Cluster"]) == "cluster" + assert tif.choose_indexer(["Individual", "DataSet"]) == "dataset" assert tif.choose_indexer(["Individual", "Anatomy"]) == "anatomical_ind" diff --git a/src/test/test_transcriptomics.py b/src/test/test_transcriptomics.py index 3a2d22e..1a8bcf9 100644 --- a/src/test/test_transcriptomics.py +++ b/src/test/test_transcriptomics.py @@ -13,6 +13,9 @@ import unittest import pandas as pd from vfbquery import vfb_queries as vfb +from .scrnaseq_fixtures import (SCRNASEQ_ID_PREFIXES, anatomy_with_clusters, + cluster_with_expression, dataset_with_clusters, + gene_with_clusters) class TranscriptomicsQueriesTest(unittest.TestCase): @@ -27,7 +30,17 @@ class TranscriptomicsQueriesTest(unittest.TestCase): CLUSTER_ID = "FBlc0006181" # an scRNAseq cluster GENE_ID = "FBgn0283521" # a gene expressed across clusters DATASET_ID = "FBlc0004785" # an scRNAseq dataset - + + @classmethod + def setUpClass(cls): + # The scRNAseq loads are moving from FlyBase FBlc ids to vfb-scRNAseq's + # VFBseq ids, study by study: keep each fixture while it still has data, + # else use a live term that does (scrnaseq_fixtures). + cls.ANATOMY_WITH_SCRNASEQ = anatomy_with_clusters(cls.ANATOMY_WITH_SCRNASEQ) + cls.CLUSTER_ID = cluster_with_expression(cls.CLUSTER_ID) + cls.GENE_ID = gene_with_clusters(cls.GENE_ID) + cls.DATASET_ID = dataset_with_clusters(cls.DATASET_ID) + def test_anatomy_scrnaseq_basic_dataframe(self): """Test anatScRNAseqQuery returns DataFrame""" result = vfb.get_anatomy_scrnaseq(self.ANATOMY_WITH_SCRNASEQ, return_dataframe=True) @@ -40,10 +53,11 @@ def test_anatomy_scrnaseq_basic_dataframe(self): self.assertIn('dataset', result.columns) self.assertIn('pubs', result.columns) - # Cluster IDs are FlyBase library-cluster ids (FBlc...). + # Cluster IDs are FlyBase library-cluster ids (FBlc..., old loads) or + # vfb-scRNAseq ids (VFBseq_..., new loads). for idx, row in result.iterrows(): - self.assertTrue(row['id'].startswith('FBlc'), - f"Cluster ID should start with FBlc, got: {row['id']}") + self.assertTrue(row['id'].startswith(SCRNASEQ_ID_PREFIXES), + f"Cluster ID should start with FBlc or VFBseq_, got: {row['id']}") def test_anatomy_scrnaseq_formatted_output(self): """Test anatScRNAseqQuery returns properly formatted dict""" diff --git a/src/vfbquery/combine.py b/src/vfbquery/combine.py index 3df3ff9..da2ff98 100644 --- a/src/vfbquery/combine.py +++ b/src/vfbquery/combine.py @@ -390,10 +390,11 @@ def id_column(headers, rows=()): #: nothing depends on it for correctness. NAMESPACES = [ ("VFBexp_", "expression pattern"), + ("VFBseq_", "scRNAseq dataset / sample / assay / clustering / cluster"), ("VFBc_", "channel"), ("VFB_", "individual (a single imaged or reconstructed specimen)"), ("FBbt_", "anatomy class (a cell type or region)"), - ("FBlc_", "dataset / transcriptomic cluster"), + ("FBlc", "dataset / transcriptomic cluster"), ("FBgn_", "gene"), ("FBal_", "allele"), ("FBti_", "transposon insertion"), diff --git a/src/vfbquery/term_info_fallback.py b/src/vfbquery/term_info_fallback.py index 7b4acdf..6fe72ce 100644 --- a/src/vfbquery/term_info_fallback.py +++ b/src/vfbquery/term_info_fallback.py @@ -289,7 +289,12 @@ def fallback_unavailable_reason(): #: Clusters, which ClusterTermInfoQueryIndexer owns, so this list only applies #: on the anatomical branch. Applying it globally -- as the first version of #: this module did -- would have refused to rebuild any Cluster document. -ANATOMICAL_EXCLUDED_ID_PREFIXES = ("VFBc_", "FBlc", "SAMN", "VFB_internal") +#: ``VFBseq_`` is the vfb-scRNAseq loads' namespace, which replaces FBlc for +#: scRNAseq Datasets, Samples, Assays, Clusterings and Clusters: its Samples, +#: Assays and Clusterings stay out of the anatomical index as their FBlc +#: predecessors did (its Datasets and Clusters take the dataset / cluster +#: branches before this list is consulted). +ANATOMICAL_EXCLUDED_ID_PREFIXES = ("VFBc_", "FBlc", "VFBseq_", "SAMN", "VFB_internal") def choose_indexer(labels): diff --git a/tests/test_combine.py b/tests/test_combine.py index f21399d..a35df1f 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -216,6 +216,10 @@ def test_namespaces_are_named_in_plain_words(): assert c.namespace_of("FBgn_0000123") == "gene" # Longest prefix wins: VFBexp_ is not a VFB_ individual. assert c.namespace_of("VFBexp_FBtp0121686") == "expression pattern" + # scRNAseq datasets / clusters: FlyBase FBlc ids (old loads, no underscore) + # and vfb-scRNAseq VFBseq_ ids (new loads; not a VFB_ individual either). + assert c.namespace_of("FBlc0004785") == "dataset / transcriptomic cluster" + assert c.namespace_of("VFBseq_00003217").startswith("scRNAseq") # ---------------------------------------------------------------------------