diff --git a/clients/vfbquery-client/src/vfbquery_client/client.py b/clients/vfbquery-client/src/vfbquery_client/client.py index 2159832..4490133 100644 --- a/clients/vfbquery-client/src/vfbquery_client/client.py +++ b/clients/vfbquery-client/src/vfbquery_client/client.py @@ -159,7 +159,7 @@ def _raise_server_warnings(path: str, payload) -> None: #: uses ``connections``. Without the second name its envelope falls through to #: the "a dict is one row" branch below and the caller gets a 1x3 frame of #: nested lists instead of the connectivity table. - _ROW_KEYS = ("rows", "connections") + _ROW_KEYS = ("rows", "connections", "neurotransmitters") @classmethod def _to_df(cls, payload) -> pd.DataFrame: @@ -321,6 +321,44 @@ def get_neuron_connectivity(self, neuron_id: str) -> pd.DataFrame: return self._to_df(self._get("run_query", id=neuron_id, query_type="NeuronNeuronConnectivityQuery")) + def get_predicted_neurotransmitters(self, neuron_type: str, + aggregate: bool = True, + split_by_dataset: bool = False, + exclude_dbs: Optional[Iterable[str]] = None, + min_confidence: float = 0.0, + ) -> pd.DataFrame: + """Predicted neurotransmitter(s) for a type (GET /get_predicted_neurotransmitters). + + **A type includes its subclasses**, as on + :meth:`get_connected_neurons_by_type`. Predictions come from per-instance + ``capable_of`` edges carrying a confidence. + + ``aggregate`` (default) returns flat per-class rows with ``instances``, + ``percent_of_class`` and ``mean_confidence``; ``aggregate=False`` returns + one row per neuron. ``split_by_dataset`` (aggregate only) adds a + ``dataset`` column and one row per dataset. The neurotransmitter is a GO + secretion term (``nt_id``/``nt_label``). ``exclude_dbs`` behaves as on + :meth:`get_connected_neurons_by_type` (``[]`` for all datasets). + """ + dbs = None if exclude_dbs is None else ",".join(exclude_dbs) + return self._to_df(self._get("get_predicted_neurotransmitters", + neuron_type=neuron_type, + aggregate=str(aggregate).lower(), + split_by_dataset=str(split_by_dataset).lower(), + exclude_dbs=dbs, + min_confidence=min_confidence)) + + def get_known_neurotransmitters(self, neuron_type: str) -> pd.DataFrame: + """Known (curated) neurotransmitter(s) for a type and its subclasses + (GET /get_known_neurotransmitters). + + Ontology classification rather than per-instance prediction, so no + confidence. One row per ``(cell_type, nt)``; the neurotransmitter is a GO + secretion term. Empty when the ontology asserts none. + """ + return self._to_df(self._get("get_known_neurotransmitters", + neuron_type=neuron_type)) + # ---- similarity ------------------------------------------------------ def get_similar_neurons(self, neuron_id: str) -> pd.DataFrame: """NBLAST morphological matches (run_query SimilarMorphologyTo).""" diff --git a/docs/http-api.md b/docs/http-api.md index fa3221d..75853cd 100644 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -20,6 +20,8 @@ Call the service host directly when you need to see the current answer rather th | `/get_term_info` | Everything VFB holds about one term: name, synonyms, definition, relationships, images, xrefs, NT predictions, publications. | | `/run_query` | Any of the ~40 named query types — instances, subclasses, parts, connectivity, NBLAST, expression, single-cell. The workhorse. | | `/query_connectivity` | Connectivity between two *types*, aggregated, across connectome datasets. | +| `/get_predicted_neurotransmitters` | Predicted neurotransmitter(s) for a type, per instance or aggregated (with mean confidence), optionally split by dataset. | +| `/get_known_neurotransmitters` | Known (curated) neurotransmitter(s) for a type and its subclasses, from ontology classification. | | `/search` | Free-text search over the ontology, ranked the way the website ranks it. | | `/xref` | VFB id ↔ external accession, both directions. | | `/facets` | Every type name `/search`'s type filters accept, with term counts. | @@ -197,6 +199,44 @@ The cost of this default is that a plain query does not reproduce a published he do that, name the dataset you want by excluding the others, or pass `exclude_dbs=` to get everything and deduplicate yourself. `/list_connectome_datasets` gives the symbols. +## `/get_predicted_neurotransmitters` + +``` +GET /get_predicted_neurotransmitters?neuron_type=Tm9 +``` + +Predicted neurotransmitter(s) for a neuron type — itself or any subclass — from per-instance +prediction edges (an asserted `capable_of` to a GO secretion term, carrying a confidence). Only +neurons the pipeline could predict contribute; those it could not (e.g. too few presynapses) are +absent. + +| Parameter | | +|---|---| +| `neuron_type` | **Required.** Neuron type label, synonym or FBbt id. Means itself *and its subclasses*, as on `/query_connectivity`. | +| `aggregate` | `true` (default) returns flat per-class rows `{cell_type_id, cell_type, nt_id, nt_label, instances, percent_of_class, mean_confidence}`; `false` returns per-instance rows `{cell_type_id, cell_type, neuron_id, neuron_name, nt_id, nt_label, confidence, references, dataset}`. | +| `split_by_dataset` | `true` (aggregate only) emits one row per `(cell_type, nt, dataset)` and adds a `dataset` column, so agreement across connectomes is visible. | +| `exclude_dbs` | As on `/query_connectivity`; defaults to `hb,fafb`. Echoed back as `excluded_dbs`. | +| `min_confidence` | Drop predictions below this confidence (0..1). | +| `force_refresh` | Bypass the cache. | + +The neurotransmitter is reported as its **GO secretion term** (`nt_id`, e.g. `GO_0014055` +"acetylcholine secretion, neurotransmission") — the same id space as `/get_known_neurotransmitters`. +Because the pipeline assigns a single neurotransmitter per neuron, `percent_of_class` sums to ~100% +across the neurotransmitters of a cell type. + +## `/get_known_neurotransmitters` + +``` +GET /get_known_neurotransmitters?neuron_type=Tm9 +``` + +Known (curated) neurotransmitter(s) for a neuron type and its subclasses, from the ontology's +classification rather than per-instance predictions — so **no confidence**. One row per +`(cell_type, nt)`: `{cell_type_id, cell_type, nt_id, nt_label}`, with `nt_id` a GO secretion term. +Empty when the ontology asserts no neurotransmitter for the type. Read from the materialised +`SUBCLASSOF` + `capable_of` structure (a neuron class links to a neurotransmitter-type class that +carries a `capable_of` edge to the GO term), which is fast and independent of live reasoning. + ## `/get_hierarchy` ``` diff --git a/src/test/test_vfb_connectivity.py b/src/test/test_vfb_connectivity.py index 2cf7191..ab88dfb 100644 --- a/src/test/test_vfb_connectivity.py +++ b/src/test/test_vfb_connectivity.py @@ -19,7 +19,10 @@ """ import pytest -from vfbquery.vfb_connectivity import list_connectome_datasets, query_connectivity +from vfbquery.vfb_connectivity import ( + list_connectome_datasets, query_connectivity, + get_predicted_neurotransmitters, get_known_neurotransmitters, +) #: A small, stable pair: 8 and 30 connectivity individuals respectively, one #: class each, no subclasses. Cheap enough to query several times — the two @@ -335,3 +338,109 @@ def test_nonexistent_type_returns_warning(self): def test_no_types_raises_error(self): with pytest.raises(ValueError, match="At least one"): query_connectivity() + + +# --------------------------------------------------------------------------- +# Neurotransmitter queries +# +# Tm9 is the stable fixture: a well-characterised, uncontroversially +# cholinergic optic-lobe type (FBbt_00003797) with two subtypes (Tm9a/Tm9b), +# so it exercises both the aggregation and the subclass behaviour while staying +# small — the aggregate is a couple of rows and the known-NT answer a handful. +# --------------------------------------------------------------------------- +NT_TYPE = "FBbt_00003797" # transmedullary neuron Tm9 +NT_TYPE_LABEL = "transmedullary neuron Tm9" +ACH = "GO_0014055" # acetylcholine secretion, neurotransmission + + +@pytest.fixture(scope="module") +def predicted_tm9(): + return get_predicted_neurotransmitters(NT_TYPE) + + +@pytest.fixture(scope="module") +def known_tm9(): + return get_known_neurotransmitters(NT_TYPE) + + +class TestPredictedNeurotransmitters: + @pytest.mark.integration + def test_tm9_is_predominantly_cholinergic(self, predicted_tm9): + rows = predicted_tm9["neurotransmitters"] + assert rows, "expected at least one predicted NT for Tm9" + # Rows are sorted per cell type by descending instances, so the first + # row for Tm9 is its dominant prediction. + top = max(rows, key=lambda r: r["instances"]) + assert top["nt_id"] == ACH + assert top["percent_of_class"] >= 90 + + @pytest.mark.integration + def test_aggregate_row_shape(self, predicted_tm9): + for r in predicted_tm9["neurotransmitters"]: + assert r["nt_id"].startswith("GO_") + assert r["nt_label"] + assert isinstance(r["instances"], int) and r["instances"] > 0 + assert 0 <= r["percent_of_class"] <= 100 + assert r["mean_confidence"] is None or 0.0 <= r["mean_confidence"] <= 1.0 + # aggregate (unsplit) rows carry no dataset column + assert "dataset" not in r + + @pytest.mark.integration + def test_per_instance_shape(self): + result = get_predicted_neurotransmitters(NT_TYPE, aggregate=False) + rows = result["neurotransmitters"] + assert rows + r = rows[0] + assert r["neuron_id"] and r["nt_id"].startswith("GO_") + assert r["confidence"] is None or 0.0 <= r["confidence"] <= 1.0 + assert "dataset" in r + + @pytest.mark.integration + def test_split_by_dataset_adds_dataset_column(self): + result = get_predicted_neurotransmitters(NT_TYPE, split_by_dataset=True) + rows = result["neurotransmitters"] + assert rows + assert all("dataset" in r for r in rows) + + @pytest.mark.integration + def test_min_confidence_filters(self, predicted_tm9): + strict = get_predicted_neurotransmitters(NT_TYPE, aggregate=False, + min_confidence=0.99) + default = get_predicted_neurotransmitters(NT_TYPE, aggregate=False) + assert strict["count"] <= default["count"] + assert all(r["confidence"] is None or r["confidence"] >= 0.99 + for r in strict["neurotransmitters"]) + + @pytest.mark.integration + def test_nonexistent_type_returns_warning(self): + result = get_predicted_neurotransmitters( + "xyzzy_nonexistent_neuron_type_99999") + assert result["count"] == 0 + assert len(result["warnings"]) > 0 + + +class TestKnownNeurotransmitters: + @pytest.mark.integration + def test_tm9_known_cholinergic(self, known_tm9): + pairs = {(r["cell_type_id"], r["nt_id"]) + for r in known_tm9["neurotransmitters"]} + assert (NT_TYPE, ACH) in pairs + + @pytest.mark.integration + def test_includes_subclasses(self, known_tm9): + # Tm9a / Tm9b are subtypes of Tm9 and should appear as their own rows. + cell_types = {r["cell_type_id"] for r in known_tm9["neurotransmitters"]} + assert len(cell_types) > 1 + + @pytest.mark.integration + def test_row_shape(self, known_tm9): + for r in known_tm9["neurotransmitters"]: + assert set(r) == {"cell_type_id", "cell_type", "nt_id", "nt_label"} + assert r["nt_id"].startswith("GO_") + + @pytest.mark.integration + def test_nonexistent_type_returns_warning(self): + result = get_known_neurotransmitters( + "xyzzy_nonexistent_neuron_type_99999") + assert result["count"] == 0 + assert len(result["warnings"]) > 0 diff --git a/src/vfbquery/__init__.py b/src/vfbquery/__init__.py index ed77cd9..23c9c7e 100644 --- a/src/vfbquery/__init__.py +++ b/src/vfbquery/__init__.py @@ -2,7 +2,9 @@ from .solr_result_cache import get_solr_cache from .flybase_stocks import resolve_entity, find_stocks from .flybase_combo_pubs import resolve_combination, find_combo_publications -from .vfb_connectivity import list_connectome_datasets, query_connectivity +from .vfb_connectivity import (list_connectome_datasets, query_connectivity, + get_predicted_neurotransmitters, + get_known_neurotransmitters) from .graph_builder import build_graph, batch_lookup_ids from .catmaid_client import (catmaid, CatmaidInstance, list_catmaid_instances, list_catmaid_commands, run_catmaid_command) diff --git a/src/vfbquery/api_docs.py b/src/vfbquery/api_docs.py index b566254..9e48207 100644 --- a/src/vfbquery/api_docs.py +++ b/src/vfbquery/api_docs.py @@ -224,6 +224,49 @@ FORCE_REFRESH_PARAM, ], }, + { + "path": "/get_predicted_neurotransmitters", + "summary": "Predicted neurotransmitter(s) for a neuron type", + "description": ( + "Predicted neurotransmitters for a neuron type (or any " + "subclass), from per-instance prediction edges. Aggregated " + "to flat per-class rows by default; optionally split by " + "dataset. Reported as GO secretion terms with mean " + "confidence."), + "params": [ + {"name": "neuron_type", "required": True, + "doc": "Neuron type (label, synonym or FBbt id)", + "example": "Tm9"}, + {"name": "aggregate", + "doc": "false returns per-instance rows (default true " + "aggregates to the class)", "example": ""}, + {"name": "split_by_dataset", + "doc": "true adds a dataset column and one row per dataset", + "example": ""}, + {"name": "exclude_dbs", + "doc": "Datasets to leave out (comma-separated)", + "example": ""}, + {"name": "min_confidence", + "doc": "Drop predictions below this confidence (0..1)", + "example": ""}, + FORCE_REFRESH_PARAM, + ], + }, + { + "path": "/get_known_neurotransmitters", + "summary": "Known (curated) neurotransmitter(s) for a neuron type", + "description": ( + "Curated neurotransmitters for a neuron type and its " + "subclasses, from ontology classification (no confidence). " + "One row per (cell type, GO secretion term); empty when the " + "ontology asserts none."), + "params": [ + {"name": "neuron_type", "required": True, + "doc": "Neuron type (label, synonym or FBbt id)", + "example": "Tm9"}, + FORCE_REFRESH_PARAM, + ], + }, ], }, { diff --git a/src/vfbquery/ha_api.py b/src/vfbquery/ha_api.py index 0d534d2..a74aa91 100644 --- a/src/vfbquery/ha_api.py +++ b/src/vfbquery/ha_api.py @@ -23,6 +23,8 @@ GET /find_combo_publications?id= GET /list_connectome_datasets GET /query_connectivity?upstream_type=&downstream_type= + GET /get_predicted_neurotransmitters?neuron_type=[&aggregate=&split_by_dataset=&exclude_dbs=&min_confidence=] + GET /get_known_neurotransmitters?neuron_type= GET /get_hierarchy?id=[&relationship=&direction=&max_depth=] GET /search?query= # canonical website search GET /facets[?contains=] # type names /search accepts @@ -498,6 +500,7 @@ def snapshot(self): "/resolve_entity", "/find_stocks", "/resolve_combination", "/find_combo_publications", "/list_connectome_datasets", "/query_connectivity", + "/get_predicted_neurotransmitters", "/get_known_neurotransmitters", "/search", "/facets", "/xref", "/combine", "/get_hierarchy", "/catmaid", }) @@ -813,6 +816,28 @@ def _run_query_connectivity(upstream_type, downstream_type, weight, ) +def _run_get_predicted_neurotransmitters(neuron_type, aggregate, split_by_dataset, + exclude_dbs, min_confidence, + force_refresh=False): + """Execute get_predicted_neurotransmitters in a worker process.""" + return _vfb.get_predicted_neurotransmitters( + neuron_type=neuron_type, + aggregate=aggregate, + split_by_dataset=split_by_dataset, + exclude_dbs=exclude_dbs, + min_confidence=min_confidence, + force_refresh=force_refresh, + ) + + +def _run_get_known_neurotransmitters(neuron_type, force_refresh=False): + """Execute get_known_neurotransmitters in a worker process.""" + return _vfb.get_known_neurotransmitters( + neuron_type=neuron_type, + force_refresh=force_refresh, + ) + + # --------------------------------------------------------------------------- # Graph post-processing — mapping from query function name to graph converter # --------------------------------------------------------------------------- @@ -1114,6 +1139,27 @@ def _query_int(request, name, default, minimum=None, maximum=None): return value +def _query_float(request, name, default, minimum=None, maximum=None): + """Read a float query parameter, or raise :class:`BadParam`. + + The float analogue of :func:`_query_int` — used for ``min_confidence``, + where a bad value changes what is asked and so is rejected rather than + silently defaulted. Blank is treated as absent. + """ + raw = request.query.get(name) + if raw is None or not str(raw).strip(): + return default + try: + value = float(raw) + except (TypeError, ValueError): + raise BadParam("%s must be a number (got %r)" % (name, raw)) + if minimum is not None and value < minimum: + raise BadParam("%s must be at least %s (got %s)" % (name, minimum, value)) + if maximum is not None and value > maximum: + raise BadParam("%s must be at most %s (got %s)" % (name, maximum, value)) + return value + + #: Spellings of "yes" accepted for a boolean query parameter, matching what the #: existing ``force_refresh`` parsing already accepts across this module. _TRUE_VALUES = ("true", "1", "yes", "on") @@ -2189,6 +2235,99 @@ def post_fn(result): ) +_PREDICTED_NT_PARAMS = frozenset({ + "neuron_type", "aggregate", "split_by_dataset", "exclude_dbs", + "min_confidence", "force_refresh", +}) + +_KNOWN_NT_PARAMS = frozenset({"neuron_type", "force_refresh"}) + + +async def handle_get_predicted_neurotransmitters(request): + """GET /get_predicted_neurotransmitters?neuron_type=X&aggregate=true&split_by_dataset=false&exclude_dbs=hb,fafb&min_confidence=0 + + Predicted neurotransmitter(s) for a neuron type — itself or any subclass — + from per-instance ``capable_of`` prediction edges (those carrying a + confidence). ``aggregate`` (default true) returns flat per-class rows; + ``split_by_dataset`` adds a ``dataset`` column so cross-connectome agreement + is visible; ``min_confidence`` drops low-confidence predictions. + ``exclude_dbs`` behaves exactly as on ``/query_connectivity`` (defaults to + ``DEFAULT_EXCLUDE_DBS``; pass empty for all datasets), and the datasets left + out are echoed back as ``excluded_dbs``. + """ + neuron_type = request.query.get("neuron_type") or None + if neuron_type is None: + return web.json_response({"error": "neuron_type required"}, status=400) + aggregate = _query_flag(request, "aggregate", default=True) + split_by_dataset = _query_flag(request, "split_by_dataset") + try: + min_confidence = _query_float( + request, "min_confidence", 0.0, minimum=0.0, maximum=1.0) + except BadParam as exc: + return web.json_response({"error": str(exc)}, status=400) + + exclude_dbs_raw = request.query.get("exclude_dbs") + if exclude_dbs_raw is not None: + exclude_dbs = [s.strip() for s in exclude_dbs_raw.split(",") if s.strip()] + else: + from .vfb_connectivity import DEFAULT_EXCLUDE_DBS + exclude_dbs = list(DEFAULT_EXCLUDE_DBS) + force_refresh = _force_refresh_requested(request) + + warnings = [] + if exclude_dbs: + try: + exclude_dbs, rewritten = _resolve_exclude_dbs( + exclude_dbs, await _connectome_vocabulary(request.app)) + except BadParam as exc: + return web.json_response({"error": str(exc)}, status=400) + if rewritten: + warnings.append( + "exclude_dbs %s resolved to %s" + % (", ".join(repr(v) for v in rewritten), + ", ".join(repr(v) for v in exclude_dbs))) + + def post_fn(result): + if not isinstance(result, dict): + return result + result = dict(result) + result["excluded_dbs"] = list(exclude_dbs) + if warnings: + result["warnings"] = list(result.get("warnings") or []) + warnings + return result + + key = ("predicted_neurotransmitters:%s:%s:%s:%s:%s" + % (neuron_type, aggregate, split_by_dataset, min_confidence, + exclude_dbs)) + if force_refresh: + request.app["result_cache"].invalidate(key) + return await _dispatch_to_pool( + request, key, _run_get_predicted_neurotransmitters, + neuron_type, aggregate, split_by_dataset, exclude_dbs, min_confidence, + force_refresh, post_fn=post_fn, known_params=_PREDICTED_NT_PARAMS, + ) + + +async def handle_get_known_neurotransmitters(request): + """GET /get_known_neurotransmitters?neuron_type=X + + Known (curated) neurotransmitter(s) for a neuron type and its subclasses, + read from ontology subsumption (no confidence). One row per + ``(cell_type, nt)``; empty when the ontology asserts none. + """ + neuron_type = request.query.get("neuron_type") or None + if neuron_type is None: + return web.json_response({"error": "neuron_type required"}, status=400) + force_refresh = _force_refresh_requested(request) + key = "known_neurotransmitters:%s" % (neuron_type,) + if force_refresh: + request.app["result_cache"].invalidate(key) + return await _dispatch_to_pool( + request, key, _run_get_known_neurotransmitters, + neuron_type, force_refresh, known_params=_KNOWN_NT_PARAMS, + ) + + #: Query-string keys both hierarchy handlers read. `depth` and `relation` are #: the near misses this catches. _HIERARCHY_PARAMS = frozenset({"id", "relationship", "direction", "max_depth"}) @@ -4095,6 +4234,8 @@ def create_app(max_workers=None, max_concurrent=None, max_queue_depth=None, app.router.add_get("/find_combo_publications", handle_find_combo_publications) app.router.add_get("/list_connectome_datasets", handle_list_connectome_datasets) app.router.add_get("/query_connectivity", handle_query_connectivity) + app.router.add_get("/get_predicted_neurotransmitters", handle_get_predicted_neurotransmitters) + app.router.add_get("/get_known_neurotransmitters", handle_get_known_neurotransmitters) app.router.add_get("/get_hierarchy", handle_get_hierarchy) app.router.add_get("/get_hierarchy_html", handle_get_hierarchy_html) diff --git a/src/vfbquery/vfb_connectivity.py b/src/vfbquery/vfb_connectivity.py index 67271d2..7d02db7 100644 --- a/src/vfbquery/vfb_connectivity.py +++ b/src/vfbquery/vfb_connectivity.py @@ -965,3 +965,398 @@ def _build_connectivity_cypher(upstream_ids, downstream_ids, weight, ) return " \n\n".join(clauses) + + +# --------------------------------------------------------------------------- +# Neurotransmitter queries +# --------------------------------------------------------------------------- +# +# Two views of the same biology, kept deliberately separate because their +# evidence differs: +# +# * **Predicted** (:func:`get_predicted_neurotransmitters`) is per *instance* -- +# an asserted ``capable_of`` (RO_0002215) edge from a reconstructed neuron to +# a GO neurotransmitter-secretion term (a descendant of ``GO_0007269``), set by +# the prediction pipeline (vfb-neurotransmitter-predictions, from neuprint +# ``predictedNt`` / Eckstein ``conf_nt`` / Codex). The GO target -- not the +# presence of a confidence -- is what identifies the edge as a +# *neurotransmitter*; the ``confidence_value`` is what identifies it as a +# *prediction*, so this function requires both. An NT edge without a confidence +# is curated/verified, not predicted, and belongs to the known query. +# * **Known** (:func:`get_known_neurotransmitters`) is per *class* -- a curated +# ontology classification that a neuron type is capable of a neurotransmitter +# secretion, read from the materialised ``SUBCLASSOF`` + ``capable_of`` Neo4j +# structure (not node labels, and not live reasoning), and carrying no +# confidence. +# +# Both report the transmitter as its **GO secretion term** (``GO_…``), so the +# two share one id space. + +#: GO 'neurotransmitter secretion' -- the single semantic anchor for what counts +#: as a neurotransmitter. Its subclasses are the specific secretion processes +#: (acetylcholine, GABA, glutamate, and so on); anchoring on this one root rather +#: than a hard-coded transmitter list means new neurotransmitters are picked up +#: from the ontology automatically. +_NT_SECRETION_ROOT = "GO_0007269" + +_NT_GO_TERMS = None + + +def _nt_go_terms(nc): + """GO neurotransmitter-secretion terms, as an ordered ``{short_form: label}``. + + These are the descendants of :data:`_NT_SECRETION_ROOT` (``GO_0007269`` + 'neurotransmitter secretion'), read from the **materialised ``SUBCLASSOF`` + hierarchy in Neo4j** -- fast, and independent of Owlery (whose subsumption of + this GO root times out). This is the id space both NT functions report in, + and -- crucially -- the set that decides which ``capable_of`` edges are + neurotransmitters: ``capable_of`` (RO_0002215) is *also* used for + non-neurotransmitter neuron functions (feeding behaviour, locomotion, light + perception, and so on), so membership of this set, **not** the presence of a + ``confidence_value``, is what identifies a neurotransmitter edge. Anchoring + on the GO root avoids both a hard-coded transmitter list and any assumption + about which edges carry confidence. Memoised per process -- the set only + changes on a KB reload. + """ + global _NT_GO_TERMS + if _NT_GO_TERMS is None: + results = nc.commit_list([ + "MATCH (g:Class)-[:SUBCLASSOF*0..]->" + f"(:Class {{short_form: {_cypher_str(_NT_SECRETION_ROOT)}}}) " + "WHERE g.short_form STARTS WITH 'GO_' " + "RETURN DISTINCT g.short_form AS id, g.label AS label ORDER BY g.label" + ]) + _NT_GO_TERMS = {r["id"]: (r.get("label") or r["id"]) + for r in dict_cursor(results) if r.get("id")} + return _NT_GO_TERMS + + +# ---- predicted neurotransmitters (per-instance, with confidence) ---------- + +def _aggregate_predictions(per_instance, split_by_dataset): + """Aggregate per-instance predictions to flat per-class rows. + + One row per ``(cell_type, nt)``, or per ``(cell_type, nt, dataset)`` when + ``split_by_dataset`` is true (which adds a ``dataset`` column). Because the + pipeline assigns a single neurotransmitter per neuron, ``percent_of_class`` + (of the cell type's prediction-bearing neurons) sums to ~100% across the NTs + of a cell type. ``mean_confidence`` is the mean over the neurons in the row. + """ + from collections import defaultdict + + groups = {} + denom = defaultdict(set) # (cell_type[, dataset]) -> distinct neurons + for r in per_instance: + ct, nt, ds = r["cell_type_id"], r["nt_id"], r["dataset"] + gkey = (ct, nt, ds) if split_by_dataset else (ct, nt) + dkey = (ct, ds) if split_by_dataset else (ct,) + g = groups.get(gkey) + if g is None: + g = groups[gkey] = { + "cell_type_id": ct, "cell_type": r["cell_type"], + "nt_id": nt, "nt_label": r["nt_label"], + "dataset": ds if split_by_dataset else None, + "neurons": set(), "conf_sum": 0.0, "conf_n": 0, + } + g["neurons"].add(r["neuron_id"]) + if r["confidence"] is not None: + g["conf_sum"] += r["confidence"] + g["conf_n"] += 1 + denom[dkey].add(r["neuron_id"]) + + out = [] + for g in groups.values(): + dkey = ((g["cell_type_id"], g["dataset"]) if split_by_dataset + else (g["cell_type_id"],)) + total = len(denom[dkey]) + n = len(g["neurons"]) + row = { + "cell_type_id": g["cell_type_id"], + "cell_type": g["cell_type"], + "nt_id": g["nt_id"], + "nt_label": g["nt_label"], + "instances": n, + "percent_of_class": round((n / total) * 100) if total else 0, + "mean_confidence": (round(g["conf_sum"] / g["conf_n"], 3) + if g["conf_n"] else None), + } + if split_by_dataset: + row["dataset"] = g["dataset"] + out.append(row) + out.sort(key=lambda r: (r["cell_type"] or "", -r["instances"])) + return out + + +def _predicted_neurotransmitters_uncached(neuron_type, aggregate=True, + split_by_dataset=False, + exclude_dbs=None, min_confidence=0.0): + """Compute predicted neurotransmitters directly from Neo4j (no caching).""" + if exclude_dbs is None: + exclude_dbs = list(DEFAULT_EXCLUDE_DBS) + + nc = _get_nc() + warnings = [] + + try: + class_id = _resolve_neuron_type_label(nc, neuron_type, notes=warnings) + except ValueError as e: + warnings.append(str(e)) + return {"neurotransmitters": [], "warnings": warnings, "count": 0, + "resolved": {}} + + class_label, ids, instances = _subclass_closure(nc, class_id) + if len(ids) > MAX_SUBCLASS_IDS: + warnings.append( + f"'{class_label}' ({class_id}) has {len(ids)} subclasses, over the " + f"{MAX_SUBCLASS_IDS} limit; only neurons typed directly to it were " + "searched. Ask about a more specific type." + ) + ids = [class_id] + resolved = {"query": neuron_type, "id": class_id, "label": class_label, + "classes_searched": len(ids), "instances": instances} + + nt_go = _nt_go_terms(nc) + if not nt_go: + return {"neurotransmitters": [], "warnings": warnings, "count": 0, + "resolved": resolved} + + dbf = _db_filter_predicate("n", exclude_dbs) if exclude_dbs else None + cypher = ( + "MATCH (c:Class:Neuron)<-[:INSTANCEOF]-" + "(n:Individual:Neuron)-[cap:capable_of]->(nt:Class)\n" + f"WHERE c.short_form IN {_id_list(ids)} " + f"AND nt.short_form IN {_id_list(list(nt_go))}\n" + # A prediction is an NT edge (GO target in the set) that carries a + # confidence. The GO filter is what makes it a neurotransmitter; the + # confidence is what makes it a *prediction*, which is what this function + # returns — an NT edge without one is curated/verified, not predicted, and + # belongs to get_known_neurotransmitters instead. + "AND EXISTS(cap.confidence_value)\n" + "AND NOT n:Deprecated" + + (f"\nAND {dbf}" if dbf else "") + + "\nOPTIONAL MATCH (n)-[:database_cross_reference]->" + "(s:Individual:Site {is_data_source:[True]})\n" + "RETURN c.short_form AS cell_type_id, c.label AS cell_type, " + "n.short_form AS neuron_id, n.label AS neuron_name, " + "nt.short_form AS nt_id, nt.label AS nt_label, " + "cap.confidence_value[0] AS confidence, " + "cap.database_cross_reference AS references, " + "s.short_form AS dataset" + ) + + per_instance = [] + seen = set() + for r in dict_cursor(nc.commit_list([cypher])): + conf = r.get("confidence") + try: + conf = float(conf) if conf is not None else None + except (TypeError, ValueError): + conf = None + if conf is not None and conf < min_confidence: + continue + # An instance is normally typed to a single neuron class in the closure, + # but dedupe defensively so a multiply-typed neuron is not counted twice. + key = (r.get("neuron_id"), r.get("nt_id"), + r.get("cell_type_id"), r.get("dataset")) + if key in seen: + continue + seen.add(key) + per_instance.append({ + "cell_type_id": r.get("cell_type_id"), + "cell_type": r.get("cell_type"), + "neuron_id": r.get("neuron_id"), + "neuron_name": r.get("neuron_name"), + "nt_id": r.get("nt_id"), + "nt_label": r.get("nt_label"), + "confidence": conf, + "references": r.get("references") or [], + "dataset": r.get("dataset"), + }) + + if not aggregate: + return {"neurotransmitters": per_instance, "warnings": warnings, + "count": len(per_instance), "resolved": resolved} + + rows = _aggregate_predictions(per_instance, split_by_dataset) + return {"neurotransmitters": rows, "warnings": warnings, + "count": len(rows), "resolved": resolved} + + +def _predicted_nt_cache_key(neuron_type, aggregate, split_by_dataset, + min_confidence, exclude_dbs): + """Composite Solr-safe cache key for a get_predicted_neurotransmitters call + (the default ``@with_solr_cache`` keys on a single id, which does not fit + this signature — same approach as :func:`_connectivity_cache_key`).""" + import hashlib + raw = (f"predicted_neurotransmitters:{neuron_type}:{aggregate}:" + f"{split_by_dataset}:{min_confidence}:{exclude_dbs}") + return hashlib.sha1(raw.encode("utf-8")).hexdigest() + + +def get_predicted_neurotransmitters(neuron_type, aggregate=True, + split_by_dataset=False, exclude_dbs=None, + min_confidence=0.0, force_refresh=False): + """Predicted neurotransmitter(s) for a neuron type, per instance or + aggregated to the class. + + A type means itself *or any of its subclasses* (see the module docstring), + so asking about "Tm9" covers the Tm9a/Tm9b subtypes too. Only neurons + carrying a prediction edge contribute -- an asserted ``capable_of`` to a + neurotransmitter-secretion GO term (a descendant of ``GO_0007269``) that also + carries a ``confidence_value`` -- so neurons the pipeline could not predict + (e.g. too few presynapses) are absent, as is any curated/verified NT edge + without a confidence (see :func:`get_known_neurotransmitters` for those). To + compare predicted against known, run both queries. + + :param neuron_type: neuron type label (e.g. "Tm9") or FBbt id. + :param aggregate: when True (default) return flat per-class rows + ``{cell_type_id, cell_type, nt_id, nt_label, instances, + percent_of_class, mean_confidence}``; when False return per-instance + rows ``{cell_type_id, cell_type, neuron_id, neuron_name, nt_id, + nt_label, confidence, references, dataset}``. + :param split_by_dataset: when True (aggregate only) emit one row per + ``(cell_type, nt, dataset)`` and add a ``dataset`` column, so agreement + across connectomes is visible; when False aggregate over all included + datasets. + :param exclude_dbs: dataset symbols to exclude; defaults to + :data:`DEFAULT_EXCLUDE_DBS`. Pass ``[]`` for every dataset, or a list + naming everything but the one connectome you want to filter to it. + :param min_confidence: drop predictions below this confidence (0..1). + :param force_refresh: bypass the Solr cache and recompute. + :return: dict with 'neurotransmitters' (list), 'warnings' (list), + 'count' (int) and 'resolved' (how the type label was interpreted). + """ + if exclude_dbs is None: + exclude_dbs = list(DEFAULT_EXCLUDE_DBS) + + from .solr_result_cache import get_solr_cache, solr_caching_disabled + if solr_caching_disabled(): + return _predicted_neurotransmitters_uncached( + neuron_type, aggregate, split_by_dataset, exclude_dbs, min_confidence + ) + + cache = get_solr_cache() + cache_key = _predicted_nt_cache_key( + neuron_type, aggregate, split_by_dataset, min_confidence, exclude_dbs + ) + if force_refresh: + cache.clear_cache_entry('predicted_neurotransmitters', cache_key) + else: + cached = cache.get_cached_result('predicted_neurotransmitters', cache_key) + if cached is not None: + return cached + + result = _predicted_neurotransmitters_uncached( + neuron_type, aggregate, split_by_dataset, exclude_dbs, min_confidence + ) + try: + if isinstance(result, dict) and result.get('count', -1) >= 0: + cache.cache_result('predicted_neurotransmitters', cache_key, result) + except Exception: + pass + return result + + +# ---- known neurotransmitters (per-class, curated, via Owlery) -------------- + +def _known_neurotransmitters_uncached(neuron_type): + """Compute known (curated) neurotransmitters directly (no caching). + + For the queried class and each subclass, report the GO neurotransmitter + terms the ontology classifies it capable of. That classification is already + **materialised in Neo4j**: a neuron class links by ``SUBCLASSOF`` to a + neurotransmitter-type class (e.g. the Cell Ontology ``cholinergic neuron``, + ``CL_0000108``) which carries a ``capable_of`` (RO_0002215) edge to the GO + secretion term. So this is one structural query over ``SUBCLASSOF`` + + ``capable_of`` edges, filtered to the neurotransmitter GO set + (:func:`_nt_go_terms`). + + This deliberately does *not* use live Owlery subsumption: reasoning + ``neuron and capable_of some `` per neurotransmitter measured ~50s + each (minutes per call), whereas the materialised structure answers in well + under a second and gives identical results. It is also not node-label + parsing — it keys on the ``capable_of`` edge and GO ids. The GO filter is + essential: every neuron also reaches ``neuron`` -> *transmission of nerve + impulse* (``GO_0019226``), which is not a neurotransmitter and is excluded + by not being in the set. + """ + nc = _get_nc() + warnings = [] + + try: + class_id = _resolve_neuron_type_label(nc, neuron_type, notes=warnings) + except ValueError as e: + warnings.append(str(e)) + return {"neurotransmitters": [], "warnings": warnings, "count": 0, + "resolved": {}} + + class_label, ids, _ = _subclass_closure(nc, class_id) + resolved = {"query": neuron_type, "id": class_id, "label": class_label, + "classes_searched": len(ids)} + + nt_go = _nt_go_terms(nc) + if not nt_go: + return {"neurotransmitters": [], "warnings": warnings, "count": 0, + "resolved": resolved} + + cypher = ( + "MATCH (c:Class:Neuron)-[:SUBCLASSOF*0..]->(x:Class)" + "-[:capable_of]->(go:Class)\n" + f"WHERE c.short_form IN {_id_list(ids)} " + f"AND go.short_form IN {_id_list(list(nt_go))}\n" + "RETURN DISTINCT c.short_form AS cell_type_id, c.label AS cell_type, " + "go.short_form AS nt_id, go.label AS nt_label" + ) + rows = [{ + "cell_type_id": r.get("cell_type_id"), + "cell_type": r.get("cell_type"), + "nt_id": r.get("nt_id"), + "nt_label": r.get("nt_label"), + } for r in dict_cursor(nc.commit_list([cypher]))] + rows.sort(key=lambda r: (r["cell_type"] or "", r["nt_label"] or "")) + return {"neurotransmitters": rows, "warnings": warnings, + "count": len(rows), "resolved": resolved} + + +def _known_nt_cache_key(neuron_type): + import hashlib + return hashlib.sha1( + f"known_neurotransmitters:{neuron_type}".encode("utf-8") + ).hexdigest() + + +def get_known_neurotransmitters(neuron_type, force_refresh=False): + """Known (curated) neurotransmitter(s) for a neuron type and its subclasses. + + Distinct from :func:`get_predicted_neurotransmitters`: this is the + ontology's curated classification (no confidence), read by Owlery + subsumption rather than from per-instance prediction edges. A type covers + itself and its subclasses, so a row appears for the queried class and for + each subclass the ontology gives a known neurotransmitter. + + :param neuron_type: neuron type label (e.g. "Tm9") or FBbt id. + :param force_refresh: bypass the Solr cache and recompute. + :return: dict with 'neurotransmitters' (list of + ``{cell_type_id, cell_type, nt_id, nt_label}``), 'warnings', 'count' + and 'resolved'. Empty when the ontology asserts no neurotransmitter. + """ + from .solr_result_cache import get_solr_cache, solr_caching_disabled + if solr_caching_disabled(): + return _known_neurotransmitters_uncached(neuron_type) + + cache = get_solr_cache() + cache_key = _known_nt_cache_key(neuron_type) + if force_refresh: + cache.clear_cache_entry('known_neurotransmitters', cache_key) + else: + cached = cache.get_cached_result('known_neurotransmitters', cache_key) + if cached is not None: + return cached + + result = _known_neurotransmitters_uncached(neuron_type) + try: + if isinstance(result, dict) and result.get('count', -1) >= 0: + cache.cache_result('known_neurotransmitters', cache_key, result) + except Exception: + pass + return result diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 7ab269a..b383292 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -5434,6 +5434,28 @@ def get_images_that_develop_from(short_form: str, return_dataframe=True, limit: solr_field='anat_image_query', query_by_label=False, query_instances=True, offset=offset) +def _capable_of_expression(target_iri: str) -> str: + """OWL class expression ``neuron and capable_of some ``. + + ```` is *neuron*, ```` is *capable of*. Used by + :func:`get_neurons_capable_of`, which dispatches it to Owlery as an + *instances* query -- which individual neurons are capable of ``target`` + (e.g. a neurotransmitter-secretion GO term). + + (The class-level converse -- which neuron *classes* are curated capable of a + neurotransmitter -- is answered in ``vfb_connectivity`` by a materialised + ``SUBCLASSOF`` + ``capable_of`` Neo4j query rather than by reasoning this + expression per term, which is far too slow live.) + + ``target_iri`` is a full IRI (e.g. ``http://purl.obolibrary.org/obo/GO_0014055``). + """ + return ( + " " + "and " + f"some <{target_iri}>" + ) + + @with_solr_cache('neurons_capable_of') def get_neurons_capable_of(short_form: str, return_dataframe=True, limit: int = -1, offset: int = 0): """ @@ -5452,7 +5474,7 @@ def get_neurons_capable_of(short_form: str, return_dataframe=True, limit: int = :param limit: maximum number of results (default -1, all) :return: Individual neurons capable of the specified process """ - owl_query = f" and some <{_short_form_to_iri(short_form)}>" + owl_query = _capable_of_expression(_short_form_to_iri(short_form)) return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_image_query', query_by_label=False, query_instances=True, offset=offset)