Rebuild a missing term_info document instead of serving nothing

Dockerfile

@@ -18,6 +18,37 @@ ARG NUMPY_SPEC=numpy
 RUN pip install --no-cache-dir --no-binary numpy \
         -Csetup-args=-Dcpu-baseline=none "${NUMPY_SPEC}"
 
+# The term_info fallback (src/vfbquery/term_info_fallback.py) rebuilds a
+# missing vfb_json document by running the bulk indexer's own query, so the
+# indexer has to be importable. It is not a package: the `precompute live
+# query results` Jenkins job clones it, clones VFB_json_schema alongside,
+# copies the schema's src into src/vfb and puts the checkout on PYTHONPATH.
+# Do exactly that, so what runs here is what runs in the bulk job.
+#
+# Pin by passing --build-arg INDEXER_REF=<sha>; the default tracks master the
+# way the Jenkins job does. Both resolved SHAs are baked into the image so
+# /status can report which schema built a given document.
+ARG INDEXER_REF=master
+ARG JSON_SCHEMA_REF=master
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git && \
+    rm -rf /var/lib/apt/lists/*
+RUN git clone --quiet https://github.com/VirtualFlyBrain/VFB_json_schema_indexer.git /opt/vfb_indexer && \
+    git -C /opt/vfb_indexer checkout --quiet "${INDEXER_REF}" && \
+    git clone --quiet https://github.com/VirtualFlyBrain/VFB_json_schema.git /opt/vfb_json_schema && \
+    git -C /opt/vfb_json_schema checkout --quiet "${JSON_SCHEMA_REF}" && \
+    mkdir -p /opt/vfb_indexer/src/vfb && \
+    cp -r /opt/vfb_json_schema/src/* /opt/vfb_indexer/src/vfb/ && \
+    printf 'VFB_INDEXER_SHA=%s\nVFB_JSON_SCHEMA_SHA=%s\n' \
+        "$(git -C /opt/vfb_indexer rev-parse --short HEAD)" \
+        "$(git -C /opt/vfb_json_schema rev-parse --short HEAD)" > /opt/vfb_versions.env && \
+    cat /opt/vfb_versions.env && \
+    rm -rf /opt/vfb_indexer/.git /opt/vfb_json_schema
+# jsonschema and tqdm are the indexer's own runtime deps that VFBquery does
+# not already have (requests and vfb_connect it does).
+RUN pip install --no-cache-dir jsonschema tqdm
+ENV PYTHONPATH=/opt/vfb_indexer
+
 # Install Python deps first (layer caching)
 COPY requirements.txt setup.py pyproject.toml README.md ./
 COPY src/ src/
@@ -35,4 +66,6 @@ EXPOSE 8080
 #   VFBQUERY_CACHE_TTL       (default: 300 seconds)
 #   VFBQUERY_SOLR_WRITE_TIMEOUT (default: 30 seconds)
 
+#   VFB_JSON_SCHEMA_SHA      (set from the build; stamped into rebuilt documents)
+
 ENTRYPOINT ["python", "-m", "vfbquery.ha_api"]

src/test/test_term_info_fallback.py

@@ -0,0 +1,165 @@
+"""Unit tests for the term_info SOLR fallback.
+
+No backend and no indexer checkout required: the parts that talk to Neo4j
+and SOLR are exercised elsewhere (live) and are deliberately not mocked
+here. What is worth pinning without a backend is the type dispatch, the
+exclusions, the write guard and the ``src`` import shim -- the pieces that
+decide *whether* the fallback acts, and that would fail silently.
+"""
+import os
+import sys
+
+import pytest
+
+from vfbquery import term_info_fallback as tif
+
+
+# ---------------------------------------------------------------------------
+# Type dispatch -- mirrors the indexers' get_parameters_query predicates
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize("labels,expected", [
+    # The case that prompted all this: a painted domain with no SOLR doc.
+    (["Entity", "Individual", "Adult", "Anatomy", "Nervous_system",
+      "Synaptic_neuropil", "Synaptic_neuropil_domain", "has_image"],
+     "anatomical_ind"),
+    # Individual sub-types are checked before the catch-all, exactly as the
+    # anatomical indexer's query excludes each of them.
+    (["Individual", "Template", "Anatomy"], "template"),
+    (["Individual", "License"], "license"),
+    (["Individual", "DataSet"], "dataset"),
+    (["Individual", "pub"], "pub"),
+    (["Individual", "Cluster"], "cluster"),
+    # Classes split three ways.
+    (["Class", "Neuron"], "neuron_class"),
+    (["Class", "Split"], "split_class"),
+    (["Class", "Anatomy"], "class"),
+])
+def test_dispatch_matches_the_indexer_populations(labels, expected):
+    assert tif.choose_indexer(labels) == expected
+
+
+def test_dispatch_declines_what_no_indexer_covers():
+    assert tif.choose_indexer(["Property"]) is None
+    assert tif.choose_indexer([]) is None
+    assert tif.choose_indexer(None) is None
+
+
+def test_template_wins_over_individual():
+    """A Template is an Individual; the template indexer must claim it, or we
+    would build an anatomical-individual document for a template."""
+    assert tif.choose_indexer(["Individual", "Template"]) == "template"
+
+
+# ---------------------------------------------------------------------------
+# Exclusions -- ids the bulk indexer never writes a document for
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize("short_form", [
+    "VFBc_00000001", "FBlc0006125", "SAMN12345678", "VFB_internal_thing",
+])
+def test_excluded_ids_are_not_built(short_form):
+    payload, doc = tif.build_term_info(short_form)
+    assert payload is None and doc is None
+
+
+# ---------------------------------------------------------------------------
+# Write guard
+# ---------------------------------------------------------------------------
+
+def test_no_write_when_the_cache_is_disabled(monkeypatch):
+    """A live-data test run must never write into the shared production
+    collection. VFB has been bitten by exactly that before."""
+    monkeypatch.setenv("VFBQUERY_CACHE_ENABLED", "false")
+    monkeypatch.setattr(tif, "_INDEXERS", {"class": object})
+    sent = []
+    monkeypatch.setattr(tif, "_send_solr_docs",
+                        lambda docs, service: sent.append((docs, service)) or True)
+    assert tif.write_term_info({"id": "FBbt_00003748"}) is False
+    assert sent == []
+
+
+def test_write_uses_the_indexers_service_name(monkeypatch):
+    monkeypatch.setenv("VFBQUERY_CACHE_ENABLED", "true")
+    monkeypatch.setattr(tif, "_INDEXERS", {"class": object})
+    sent = []
+    monkeypatch.setattr(tif, "_send_solr_docs",
+                        lambda docs, service: sent.append((list(docs), service)) or True)
+    doc = {"id": "FBbt_00003748", "term_info": {"set": "{}"}}
+    assert tif.write_term_info(doc) is True
+    assert sent == [([doc], "term_info")]
+
+
+def test_a_failing_write_is_not_fatal(monkeypatch):
+    monkeypatch.setenv("VFBQUERY_CACHE_ENABLED", "true")
+    monkeypatch.setattr(tif, "_INDEXERS", {"class": object})
+
+    def boom(docs, service):
+        raise RuntimeError("solr down")
+
+    monkeypatch.setattr(tif, "_send_solr_docs", boom)
+    assert tif.write_term_info({"id": "x"}) is False
+
+
+# ---------------------------------------------------------------------------
+# Schema version stamping
+# ---------------------------------------------------------------------------
+
+def test_schema_version_prefers_the_environment(monkeypatch):
+    monkeypatch.setenv("VFB_JSON_SCHEMA_SHA", "deadbee")
+    assert tif.schema_version() == "deadbee"
+
+
+def test_schema_version_falls_back_to_the_image_file(monkeypatch, tmp_path):
+    monkeypatch.delenv("VFB_JSON_SCHEMA_SHA", raising=False)
+    versions = tmp_path / "vfb_versions.env"
+    versions.write_text("VFB_INDEXER_SHA=aaaaaaa\nVFB_JSON_SCHEMA_SHA=bbbbbbb\n")
+    monkeypatch.setattr(tif, "VERSIONS_FILE", str(versions))
+    assert tif.schema_version() == "bbbbbbb"
+
+
+def test_schema_version_is_never_a_git_call(monkeypatch, tmp_path):
+    """query_roller.get_version_tag shells out to git and raises outside a
+    repository, which a running container never is. Whatever happens, this
+    must return a string rather than blow up mid-query."""
+    monkeypatch.delenv("VFB_JSON_SCHEMA_SHA", raising=False)
+    monkeypatch.setattr(tif, "VERSIONS_FILE", str(tmp_path / "absent"))
+    assert tif.schema_version() == "unpinned"
+
+
+# ---------------------------------------------------------------------------
+# The `src` package collision
+# ---------------------------------------------------------------------------
+
+def test_import_shim_restores_the_previous_src():
+    """VFBquery's own source directory is a regular package called `src`, the
+    same name as the indexer's. The shim must put ours back afterwards or the
+    rest of the process loses it."""
+    import src as before
+    with tif._indexer_importable():
+        assert sys.path[0] == tif.INDEXER_ROOT
+    import src as after
+    assert after is before
+    assert tif.INDEXER_ROOT not in sys.path
+
+
+def test_import_shim_restores_even_when_the_body_raises():
+    import src as before
+    with pytest.raises(ValueError):
+        with tif._indexer_importable():
+            raise ValueError("boom")
+    import src as after
+    assert after is before
+    assert tif.INDEXER_ROOT not in sys.path
+
+
+# ---------------------------------------------------------------------------
+# Degrading without the indexer
+# ---------------------------------------------------------------------------
+
+def test_reports_why_it_cannot_build(monkeypatch):
+    monkeypatch.setattr(tif, "_INDEXERS", None)
+    monkeypatch.setattr(tif, "_IMPORT_ERROR", ImportError("no indexer here"))
+    assert tif.fallback_available() is False
+    assert "no indexer here" in tif.fallback_unavailable_reason()
+    assert tif.backfill_term_info("FBbt_00003748") is None