full set of subclasses for double-ended connectivity queries

docs/http-api.md

@@ -113,7 +113,7 @@ GET /query_connectivity?upstream_type=DA1 lPN&downstream_type=Kenyon cell
 |---|---|
 | `upstream_type`, `downstream_type` | Neuron type labels, synonyms or FBbt ids. **At least one is required**; giving one asks "everything downstream of / upstream of this". |
 | `weight` | Minimum synapse count for a connection to be reported. Default 5. |
-| `group_by_class` | `true` aggregates to one row per class pair, with `pairwise_connections`, `average_weight` and `percent_connected`. Default is one row per neuron pair. |
+| `group_by_class` | `true` aggregates by class, rolled up over the subclass hierarchy: a connection counts toward **every** (upstream level, downstream level) pair up to the queried type(s), so a row appears for the queried type itself *and* for each subclass that has data — with `pairwise_connections`, `total_weight`, `average_weight` and `percent_connected`. A single connection therefore contributes to several rows, so the per-row figures do not sum to the raw connection count. This matches `vfb_connect`'s `get_connected_neurons_by_type`. Default is one row per neuron pair. |
 | `exclude_dbs` | Comma-separated datasets to leave out. Defaults to `hb,fafb` — see below. Pass `exclude_dbs=` (empty) for every dataset. A symbol (`mc`), a short_form (`male_cns_v0_9`), a label (`male-cns`) or the whole label all name the same dataset; an unrecognised one is a **400** with suggestions. |
 | `include_graph` | Attach a graph structure alongside the table. |
 | `force_refresh` | Bypass the cache. |

src/test/test_graph_builder.py

@@ -178,6 +178,10 @@ def fake_batch(ids):
         }
     import vfbquery.graph_builder as gb
     monkeypatch.setattr(gb, "batch_lookup_ids", fake_batch)
+    # Containment edges need a live graph DB; default to none so unit tests
+    # don't reach the network. Tests that exercise the compound graph override
+    # this with their own stub.
+    monkeypatch.setattr(gb, "subclass_containment_edges", lambda ids: [])
 
 
 class TestGraphFromQueryConnectivity:
@@ -205,8 +209,43 @@ def test_class_level(self, monkeypatch):
         assert len(g["nodes"]) == 2
         assert len(g["edges"]) == 1
         assert g["edges"][0]["weight"] == 5000
+        assert g["edges"][0]["relation"] == "synapsed_to"
         assert g["directed"] is True
 
+    def test_class_level_compound_graph(self, monkeypatch):
+        """Rolled-up results add containment edges, tagged distinctly from the
+        synapsed_to connectivity edges so a renderer can style them apart."""
+        _mock_batch_lookup(monkeypatch)
+        import vfbquery.graph_builder as gb
+        # Stub the graph-DB lookup: child class is a subclass of the parent.
+        monkeypatch.setattr(gb, "subclass_containment_edges", lambda ids: [
+            {"source": "FBbt_child", "target": "FBbt_parent",
+             "relation": "SUBCLASSOF", "label": "subclass of", "weight": 0,
+             **gb.SUBCLASS_EDGE_STYLE},
+        ] if "FBbt_parent" in ids and "FBbt_child" in ids else [])
+        connections = [
+            {"upstream_class": "EPG", "upstream_class_id": "FBbt_parent",
+             "downstream_class": "ExR1", "downstream_class_id": "FBbt_d1",
+             "total_upstream_count": 10, "connected_upstream_count": 9,
+             "percent_connected": 90, "pairwise_connections": 20,
+             "total_weight": 500, "average_weight": 25},
+            {"upstream_class": "EPG_PB1", "upstream_class_id": "FBbt_child",
+             "downstream_class": "ExR1", "downstream_class_id": "FBbt_d1",
+             "total_upstream_count": 4, "connected_upstream_count": 4,
+             "percent_connected": 100, "pairwise_connections": 5,
+             "total_weight": 80, "average_weight": 16},
+        ]
+        g = graph_from_query_connectivity(connections, group_by_class=True,
+                                          upstream_type="EPG",
+                                          downstream_type="ExR1")
+        rels = [e.get("relation") for e in g["edges"]]
+        assert rels.count("synapsed_to") == 2
+        assert rels.count("SUBCLASSOF") == 1
+        sub = next(e for e in g["edges"] if e["relation"] == "SUBCLASSOF")
+        assert sub["source"] == "FBbt_child" and sub["target"] == "FBbt_parent"
+        assert sub["style"] == "dashed"          # pattern-coded
+        assert sub["color"] != g["edges"][0].get("color")  # colour-coded apart
+
     def test_per_neuron(self, monkeypatch):
         _mock_batch_lookup(monkeypatch)
         connections = [
@@ -321,6 +360,32 @@ def test_filters_to_input_term_block(self, monkeypatch):
         assert g["edges"][0]["target"] == "FBbt_d1"
         assert g["edges"][0]["weight"] == 5000  # input-term block, not the subclass's 500
 
+    def test_compound_graph_tags_edges(self, monkeypatch):
+        """Partner classes are rolled up over the hierarchy, so containment
+        edges are added and tagged apart from the synapsed_to edges."""
+        _mock_batch_lookup(monkeypatch)
+        import vfbquery.graph_builder as gb
+        monkeypatch.setattr(gb, "subclass_containment_edges", lambda ids: [
+            {"source": "FBbt_d_child", "target": "FBbt_d_parent",
+             "relation": "SUBCLASSOF", "label": "subclass of", "weight": 0,
+             **gb.SUBCLASS_EDGE_STYLE},
+        ] if "FBbt_d_parent" in ids and "FBbt_d_child" in ids else [])
+        rows = [
+            {"id": "FBbt_d_parent", "query_id": "FBbt_primary",
+             "downstream_class": "[ExR1](FBbt_d_parent)",
+             "pairwise_connections": 200, "total_weight": 5000},
+            {"id": "FBbt_d_child", "query_id": "FBbt_primary",
+             "downstream_class": "[ExR1 DM4](FBbt_d_child)",
+             "pairwise_connections": 50, "total_weight": 1000},
+        ]
+        g = graph_from_downstream_class(rows, "FBbt_primary", "EPG")
+        rels = [e.get("relation") for e in g["edges"]]
+        assert rels.count("synapsed_to") == 2
+        assert rels.count("SUBCLASSOF") == 1
+        sub = next(e for e in g["edges"] if e["relation"] == "SUBCLASSOF")
+        assert sub["source"] == "FBbt_d_child" and sub["target"] == "FBbt_d_parent"
+        assert sub["style"] == "dashed"
+
 
 class TestGraphFromUpstreamClass:
     def test_basic(self, monkeypatch):

src/test/test_vfb_connectivity.py

@@ -138,6 +138,72 @@ def test_group_by_class(self):
         assert "downstream_class" in conn
 
 
+class TestQueryConnectivityRollup:
+    """Grouped queries roll up over the subclass hierarchy, so a row appears for
+    every level with data up to each named query term — the two-ended analogue
+    of the single-ended DownstreamClassConnectivity / UpstreamClassConnectivity
+    rollup. The distinguishing case is EPG <-> ExR1: across datasets EPG is
+    typed both to its parent class and per-glomerulus, and ExR1 both to its
+    parent and to DM3/DM4 lineage, so only a rollup produces the parent-to-parent
+    row. exclude_dbs=[] keeps every dataset in scope."""
+
+    #: EPG parent and ExR1 parent. EPG has per-glomerulus subclasses; ExR1 has
+    #: DM3/DM4 lineage subclasses. Both parents have directly-typed instances in
+    #: some datasets, which is what makes the parent-to-parent row real.
+    ROLLUP_UP = "FBbt_00047030"    # EPG
+    ROLLUP_DOWN = "FBbt_00003655"  # ExR1
+
+    @pytest.fixture(scope="class")
+    def rollup_result(self):
+        return query_connectivity(
+            upstream_type=self.ROLLUP_UP,
+            downstream_type=self.ROLLUP_DOWN,
+            group_by_class=True,
+            exclude_dbs=[],
+        )
+
+    @pytest.mark.integration
+    def test_top_level_query_term_row_present(self, rollup_result):
+        # The row the non-rolled-up grouping never emitted: both sides at the
+        # named query term, aggregating every level beneath them.
+        conns = rollup_result["connections"]
+        assert conns
+        assert any(
+            c["upstream_class_id"] == self.ROLLUP_UP
+            and c["downstream_class_id"] == self.ROLLUP_DOWN
+            for c in conns
+        ), "expected a rolled-up EPG->ExR1 parent-to-parent row"
+
+    @pytest.mark.integration
+    def test_finer_levels_also_present(self, rollup_result):
+        # Rollup adds levels, it does not replace them: subclass-level rows still
+        # appear alongside the parent-to-parent row.
+        conns = rollup_result["connections"]
+        assert any(
+            c["upstream_class_id"] != self.ROLLUP_UP
+            or c["downstream_class_id"] != self.ROLLUP_DOWN
+            for c in conns
+        )
+
+    @pytest.mark.integration
+    def test_parent_row_dominates_its_children(self, rollup_result):
+        # The parent-to-parent row is a set-union over every child pair, so its
+        # weight is at least that of any single child pair sharing an endpoint.
+        conns = rollup_result["connections"]
+        top = next(
+            c for c in conns
+            if c["upstream_class_id"] == self.ROLLUP_UP
+            and c["downstream_class_id"] == self.ROLLUP_DOWN
+        )
+        children = [
+            c for c in conns
+            if c["upstream_class_id"] == self.ROLLUP_UP
+            and c["downstream_class_id"] != self.ROLLUP_DOWN
+        ]
+        for child in children:
+            assert top["total_weight"] >= child["total_weight"]
+
+
 class TestQueryConnectivityWeightFiltering:
     @pytest.mark.integration
     def test_higher_weight_fewer_results(self):

src/vfbquery/graph_builder.py

@@ -17,6 +17,15 @@
 
 MAX_NODES = 80
 MAX_EDGES = 200
+
+#: Above this many class nodes, skip the subclass-containment lookup. The
+#: transitive-reduction query is fine for a focused graph (a bounded subtree,
+#: e.g. EPG→ExR1 at ~14 classes) but explodes when a one-sided rolled-up query
+#: pulls in high-level classes up to the neuron root (hundreds of classes,
+#: broad ``SUBCLASSOF`` walks) — enough to time a query out. A big flat graph
+#: gains little from containment nesting anyway, so it degrades to the plain
+#: (non-compound) graph. See :func:`subclass_containment_edges`.
+MAX_CONTAINMENT_CLASSES = 50
 GRAPH_VERSION = 1
 
 # Neurotransmitter group colours (matching VFBchat conventions)
@@ -97,6 +106,69 @@ def batch_lookup_ids(ids):
         return {}
 
 
+#: Visual encoding for containment (subclass) edges, so a renderer can tell
+#: structural ``SUBCLASSOF`` edges apart from ``synapsed_to`` connectivity edges.
+#: Dashed + muted so the connectivity edges stay the focus.
+SUBCLASS_EDGE_STYLE = {"style": "dashed", "color": "#9aa4b2"}
+
+
+def subclass_containment_edges(class_ids):
+    """Containment edges (``child`` → ``parent``) among *class_ids*, as the
+    transitive reduction restricted to the present set — each class links only
+    to its *nearest present* ancestor, so the result is a clean nesting rather
+    than an edge to every ancestor.
+
+    Used to turn a rolled-up class-connectivity graph into a compound graph:
+    the same connection appears at several hierarchy levels, and these edges
+    make that nesting explicit instead of leaving parent and child as unrelated
+    siblings. Each edge is tagged ``relation="SUBCLASSOF"`` and carries dashed/
+    muted style hints (see :data:`SUBCLASS_EDGE_STYLE`) so a renderer can style
+    it differently from the ``synapsed_to`` connectivity edges. Returns ``[]``
+    for a graph with more than :data:`MAX_CONTAINMENT_CLASSES` class nodes (the
+    query is too costly there and the nesting adds little), and on any error, so
+    an unavailable graph DB degrades to the plain (non-compound) graph.
+
+    :param class_ids: ids of the class nodes already in the graph
+    :return: list of edge dicts ``{source, target, relation, ...}``
+    """
+    ids = [i for i in dict.fromkeys(class_ids) if i]
+    if len(ids) < 2 or len(ids) > MAX_CONTAINMENT_CLASSES:
+        return []
+    id_list = str(ids)
+    try:
+        nc = Neo4jConnect()
+        cypher = (
+            "MATCH (child:Class)-[:SUBCLASSOF*1..]->(parent:Class) "
+            f"WHERE child.short_form IN {id_list} "
+            f"AND parent.short_form IN {id_list} AND child <> parent "
+            "AND NOT EXISTS { "
+            "MATCH (child)-[:SUBCLASSOF*1..]->(mid:Class)-[:SUBCLASSOF*1..]->(parent) "
+            f"WHERE mid.short_form IN {id_list} AND mid <> child AND mid <> parent "
+            "} "
+            "RETURN DISTINCT child.short_form AS child, parent.short_form AS parent"
+        )
+        results = nc.commit_list([cypher])
+        if not results:
+            return []
+        rows = dict_cursor(results)
+    except Exception:
+        return []
+    edges = []
+    for r in rows:
+        child, parent = r.get("child"), r.get("parent")
+        if not child or not parent:
+            continue
+        edges.append({
+            "source": child,
+            "target": parent,
+            "relation": "SUBCLASSOF",
+            "label": "subclass of",
+            "weight": 0,
+            **SUBCLASS_EDGE_STYLE,
+        })
+    return edges
+
+
 # ---------------------------------------------------------------------------
 # Group assignment
 # ---------------------------------------------------------------------------
@@ -167,9 +239,16 @@ def build_graph(nodes, edges, title=None, directed=True, layout="force"):
     orig_node_count = len(deduped_nodes)
     orig_edge_count = len(edges)
 
-    # Truncate edges — keep highest weight first
+    # Truncate edges — keep highest-weight connectivity first, but never drop
+    # structural (containment) edges: they carry no weight, so a plain weight
+    # sort would push them to the bottom and cut them, losing the nesting.
     if len(edges) > MAX_EDGES:
-        edges = sorted(edges, key=lambda e: e.get("weight") or 0, reverse=True)[:MAX_EDGES]
+        structural = [e for e in edges if e.get("relation") == "SUBCLASSOF"]
+        weighted = [e for e in edges if e.get("relation") != "SUBCLASSOF"]
+        budget = max(0, MAX_EDGES - len(structural))
+        weighted = sorted(weighted, key=lambda e: e.get("weight") or 0,
+                          reverse=True)[:budget]
+        edges = weighted + structural
 
     # Truncate nodes — keep those with highest degree
     if len(deduped_nodes) > MAX_NODES:
@@ -289,7 +368,14 @@ def graph_from_query_connectivity(connections, group_by_class,
                     "source": up_id,
                     "target": dn_id,
                     "weight": c.get("total_weight", 0),
+                    "relation": "synapsed_to",
                 })
+        # Compound graph: a rolled-up result carries the same connection at
+        # several class levels, so add containment edges linking each class to
+        # its nearest present ancestor. Tagged distinctly (dashed/muted) from
+        # the synapsed_to edges above so subclass structure reads as nesting
+        # rather than as more connectivity.
+        edges.extend(subclass_containment_edges(list(nodes.keys())))
     else:
         # Per-neuron results
         for c in connections:
@@ -565,8 +651,14 @@ def graph_from_downstream_class(rows, primary_id, primary_label=None):
                 "source": primary_id,
                 "target": rid,
                 "weight": weight,
+                "relation": "synapsed_to",
             })
 
+    # Partner classes are rolled up over the subclass hierarchy (the single-ended
+    # rollup walks partners to the neuron root), so add containment edges — tagged
+    # apart from the synapsed_to edges — to show that nesting rather than leaving
+    # a partner parent and its subclasses as unrelated siblings.
+    edges.extend(subclass_containment_edges([n["id"] for n in nodes]))
     disp = _node_display_label(primary_info) or primary_label or primary_id
     return build_graph(nodes, edges, title=f"Downstream of {disp}", directed=True)
 
@@ -638,7 +730,13 @@ def graph_from_upstream_class(rows, primary_id, primary_label=None):
                 "source": rid,
                 "target": primary_id,
                 "weight": weight,
+                "relation": "synapsed_to",
             })
 
+    # Partner classes are rolled up over the subclass hierarchy (the single-ended
+    # rollup walks partners to the neuron root), so add containment edges — tagged
+    # apart from the synapsed_to edges — to show that nesting rather than leaving
+    # a partner parent and its subclasses as unrelated siblings.
+    edges.extend(subclass_containment_edges([n["id"] for n in nodes]))
     disp = _node_display_label(primary_info) or primary_label or primary_id
     return build_graph(nodes, edges, title=f"Upstream of {disp}", directed=True)

src/vfbquery/owlery_client.py

@@ -65,8 +65,9 @@ def __init__(self, owlery_endpoint: str = "http://owl.virtualflybrain.org/kbs/vf
         """
         self.owlery_endpoint = owlery_endpoint.rstrip('/')
 
-    def get_subclasses(self, query: str, query_by_label: bool = True, 
-                      verbose: bool = False, direct: bool = False) -> List[str]:
+    def get_subclasses(self, query: str, query_by_label: bool = True,
+                      verbose: bool = False, direct: bool = False,
+                      timeout: int = 2400) -> List[str]:
         """
         Query Owlery for subclasses matching an OWL class expression.
 
@@ -129,7 +130,7 @@ def convert_short_form_to_iri(match):
             response = session.get(
                 f"{self.owlery_endpoint}/subclasses",
                 params=params,
-                timeout=2400
+                timeout=timeout
             )
 
             if verbose: