From 537bc77494c0f60c3ea53fae8113b5ed33a07253 Mon Sep 17 00:00:00 2001 From: Robbie Court Date: Fri, 4 Sep 2026 06:48:32 +0000 Subject: [PATCH] Rebuild a missing term_info document instead of serving nothing term_info documents are written in bulk by VFB_json_schema_indexer, driven by the `precompute live query results` Jenkins job. A record that reaches the PDB after that job last succeeded has no document at all, and the gap is not hours: build #71 started 2026-09-04, #70 was aborted, and the last success before it was #66 on 2026-06-22. VFB_00107fob ("ME_R on JRC2018Unisex") is the case that surfaced it. Its in_register_with edge, its images and its parent class are all in the PDB, but with no document it was invisible to term info -- and it silently dropped out of the medulla class page too. On a SOLR miss, run the indexer's own query for the node's type, serve the result through the deserialiser get_term_info already uses, and index it so the id is only ever slow once. Nothing here re-implements the schema: the queries, the document shape and the SOLR write are imported from the indexer, which the Dockerfile clones exactly as the Jenkins job does (including copying VFB_json_schema into src/vfb). A term_info query that changes upstream is followed on the next image build. The write is conservative: only when the id genuinely has no document, and never when the SOLR cache is disabled, so a live-data test run cannot write into the shared production collection -- a fixture doing that poisoned JRC2018U term info for five days. The indexer's atomic-update shape means the sibling precomputed fields on a document are untouched regardless. Three things worth knowing: - The miss is now detected on the SOLR result, not on the parse. term_info_parse_object skips its whole body when there are no hits and returns its initialised skeleton, which fails schema validation on Name/Id/Meta and is returned raw -- so a missing term surfaced as a truthy object with no Id rather than as None. - The indexer's top-level package is called `src`, and so is VFBquery's own source directory, which carries an __init__.py and is therefore a regular package. The running process always finds its own first, so the import needs a shim that puts the indexer root ahead of it and restores `src` afterwards. - query_roller.get_version_tag shells out to `git rev-parse` against the process working directory and raises outside a repository, which a container never is. It is pinned to the schema SHA baked into the image, which is also what a rebuilt document now records in its version field. Verified live: get_term_info("VFB_00107fob") returns a complete term info (10 queries, images on JRC2018Unisex) where it previously returned a skeleton, and refuses the write with the cache disabled. --- Dockerfile | 33 +++ src/test/test_term_info_fallback.py | 165 ++++++++++++ src/vfbquery/term_info_fallback.py | 379 ++++++++++++++++++++++++++++ src/vfbquery/vfb_queries.py | 32 +++ 4 files changed, 609 insertions(+) create mode 100644 src/test/test_term_info_fallback.py create mode 100644 src/vfbquery/term_info_fallback.py diff --git a/Dockerfile b/Dockerfile index 8e42c35..2b13f3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,37 @@ ARG NUMPY_SPEC=numpy RUN pip install --no-cache-dir --no-binary numpy \ -Csetup-args=-Dcpu-baseline=none "${NUMPY_SPEC}" +# The term_info fallback (src/vfbquery/term_info_fallback.py) rebuilds a +# missing vfb_json document by running the bulk indexer's own query, so the +# indexer has to be importable. It is not a package: the `precompute live +# query results` Jenkins job clones it, clones VFB_json_schema alongside, +# copies the schema's src into src/vfb and puts the checkout on PYTHONPATH. +# Do exactly that, so what runs here is what runs in the bulk job. +# +# Pin by passing --build-arg INDEXER_REF=; the default tracks master the +# way the Jenkins job does. Both resolved SHAs are baked into the image so +# /status can report which schema built a given document. +ARG INDEXER_REF=master +ARG JSON_SCHEMA_REF=master +RUN apt-get update && \ + apt-get install -y --no-install-recommends git && \ + rm -rf /var/lib/apt/lists/* +RUN git clone --quiet https://github.com/VirtualFlyBrain/VFB_json_schema_indexer.git /opt/vfb_indexer && \ + git -C /opt/vfb_indexer checkout --quiet "${INDEXER_REF}" && \ + git clone --quiet https://github.com/VirtualFlyBrain/VFB_json_schema.git /opt/vfb_json_schema && \ + git -C /opt/vfb_json_schema checkout --quiet "${JSON_SCHEMA_REF}" && \ + mkdir -p /opt/vfb_indexer/src/vfb && \ + cp -r /opt/vfb_json_schema/src/* /opt/vfb_indexer/src/vfb/ && \ + printf 'VFB_INDEXER_SHA=%s\nVFB_JSON_SCHEMA_SHA=%s\n' \ + "$(git -C /opt/vfb_indexer rev-parse --short HEAD)" \ + "$(git -C /opt/vfb_json_schema rev-parse --short HEAD)" > /opt/vfb_versions.env && \ + cat /opt/vfb_versions.env && \ + rm -rf /opt/vfb_indexer/.git /opt/vfb_json_schema +# jsonschema and tqdm are the indexer's own runtime deps that VFBquery does +# not already have (requests and vfb_connect it does). +RUN pip install --no-cache-dir jsonschema tqdm +ENV PYTHONPATH=/opt/vfb_indexer + # Install Python deps first (layer caching) COPY requirements.txt setup.py pyproject.toml README.md ./ COPY src/ src/ @@ -35,4 +66,6 @@ EXPOSE 8080 # VFBQUERY_CACHE_TTL (default: 300 seconds) # VFBQUERY_SOLR_WRITE_TIMEOUT (default: 30 seconds) +# VFB_JSON_SCHEMA_SHA (set from the build; stamped into rebuilt documents) + ENTRYPOINT ["python", "-m", "vfbquery.ha_api"] diff --git a/src/test/test_term_info_fallback.py b/src/test/test_term_info_fallback.py new file mode 100644 index 0000000..7814598 --- /dev/null +++ b/src/test/test_term_info_fallback.py @@ -0,0 +1,165 @@ +"""Unit tests for the term_info SOLR fallback. + +No backend and no indexer checkout required: the parts that talk to Neo4j +and SOLR are exercised elsewhere (live) and are deliberately not mocked +here. What is worth pinning without a backend is the type dispatch, the +exclusions, the write guard and the ``src`` import shim -- the pieces that +decide *whether* the fallback acts, and that would fail silently. +""" +import os +import sys + +import pytest + +from vfbquery import term_info_fallback as tif + + +# --------------------------------------------------------------------------- +# Type dispatch -- mirrors the indexers' get_parameters_query predicates +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("labels,expected", [ + # The case that prompted all this: a painted domain with no SOLR doc. + (["Entity", "Individual", "Adult", "Anatomy", "Nervous_system", + "Synaptic_neuropil", "Synaptic_neuropil_domain", "has_image"], + "anatomical_ind"), + # Individual sub-types are checked before the catch-all, exactly as the + # anatomical indexer's query excludes each of them. + (["Individual", "Template", "Anatomy"], "template"), + (["Individual", "License"], "license"), + (["Individual", "DataSet"], "dataset"), + (["Individual", "pub"], "pub"), + (["Individual", "Cluster"], "cluster"), + # Classes split three ways. + (["Class", "Neuron"], "neuron_class"), + (["Class", "Split"], "split_class"), + (["Class", "Anatomy"], "class"), +]) +def test_dispatch_matches_the_indexer_populations(labels, expected): + assert tif.choose_indexer(labels) == expected + + +def test_dispatch_declines_what_no_indexer_covers(): + assert tif.choose_indexer(["Property"]) is None + assert tif.choose_indexer([]) is None + assert tif.choose_indexer(None) is None + + +def test_template_wins_over_individual(): + """A Template is an Individual; the template indexer must claim it, or we + would build an anatomical-individual document for a template.""" + assert tif.choose_indexer(["Individual", "Template"]) == "template" + + +# --------------------------------------------------------------------------- +# Exclusions -- ids the bulk indexer never writes a document for +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("short_form", [ + "VFBc_00000001", "FBlc0006125", "SAMN12345678", "VFB_internal_thing", +]) +def test_excluded_ids_are_not_built(short_form): + payload, doc = tif.build_term_info(short_form) + assert payload is None and doc is None + + +# --------------------------------------------------------------------------- +# Write guard +# --------------------------------------------------------------------------- + +def test_no_write_when_the_cache_is_disabled(monkeypatch): + """A live-data test run must never write into the shared production + collection. VFB has been bitten by exactly that before.""" + monkeypatch.setenv("VFBQUERY_CACHE_ENABLED", "false") + monkeypatch.setattr(tif, "_INDEXERS", {"class": object}) + sent = [] + monkeypatch.setattr(tif, "_send_solr_docs", + lambda docs, service: sent.append((docs, service)) or True) + assert tif.write_term_info({"id": "FBbt_00003748"}) is False + assert sent == [] + + +def test_write_uses_the_indexers_service_name(monkeypatch): + monkeypatch.setenv("VFBQUERY_CACHE_ENABLED", "true") + monkeypatch.setattr(tif, "_INDEXERS", {"class": object}) + sent = [] + monkeypatch.setattr(tif, "_send_solr_docs", + lambda docs, service: sent.append((list(docs), service)) or True) + doc = {"id": "FBbt_00003748", "term_info": {"set": "{}"}} + assert tif.write_term_info(doc) is True + assert sent == [([doc], "term_info")] + + +def test_a_failing_write_is_not_fatal(monkeypatch): + monkeypatch.setenv("VFBQUERY_CACHE_ENABLED", "true") + monkeypatch.setattr(tif, "_INDEXERS", {"class": object}) + + def boom(docs, service): + raise RuntimeError("solr down") + + monkeypatch.setattr(tif, "_send_solr_docs", boom) + assert tif.write_term_info({"id": "x"}) is False + + +# --------------------------------------------------------------------------- +# Schema version stamping +# --------------------------------------------------------------------------- + +def test_schema_version_prefers_the_environment(monkeypatch): + monkeypatch.setenv("VFB_JSON_SCHEMA_SHA", "deadbee") + assert tif.schema_version() == "deadbee" + + +def test_schema_version_falls_back_to_the_image_file(monkeypatch, tmp_path): + monkeypatch.delenv("VFB_JSON_SCHEMA_SHA", raising=False) + versions = tmp_path / "vfb_versions.env" + versions.write_text("VFB_INDEXER_SHA=aaaaaaa\nVFB_JSON_SCHEMA_SHA=bbbbbbb\n") + monkeypatch.setattr(tif, "VERSIONS_FILE", str(versions)) + assert tif.schema_version() == "bbbbbbb" + + +def test_schema_version_is_never_a_git_call(monkeypatch, tmp_path): + """query_roller.get_version_tag shells out to git and raises outside a + repository, which a running container never is. Whatever happens, this + must return a string rather than blow up mid-query.""" + monkeypatch.delenv("VFB_JSON_SCHEMA_SHA", raising=False) + monkeypatch.setattr(tif, "VERSIONS_FILE", str(tmp_path / "absent")) + assert tif.schema_version() == "unpinned" + + +# --------------------------------------------------------------------------- +# The `src` package collision +# --------------------------------------------------------------------------- + +def test_import_shim_restores_the_previous_src(): + """VFBquery's own source directory is a regular package called `src`, the + same name as the indexer's. The shim must put ours back afterwards or the + rest of the process loses it.""" + import src as before + with tif._indexer_importable(): + assert sys.path[0] == tif.INDEXER_ROOT + import src as after + assert after is before + assert tif.INDEXER_ROOT not in sys.path + + +def test_import_shim_restores_even_when_the_body_raises(): + import src as before + with pytest.raises(ValueError): + with tif._indexer_importable(): + raise ValueError("boom") + import src as after + assert after is before + assert tif.INDEXER_ROOT not in sys.path + + +# --------------------------------------------------------------------------- +# Degrading without the indexer +# --------------------------------------------------------------------------- + +def test_reports_why_it_cannot_build(monkeypatch): + monkeypatch.setattr(tif, "_INDEXERS", None) + monkeypatch.setattr(tif, "_IMPORT_ERROR", ImportError("no indexer here")) + assert tif.fallback_available() is False + assert "no indexer here" in tif.fallback_unavailable_reason() + assert tif.backfill_term_info("FBbt_00003748") is None diff --git a/src/vfbquery/term_info_fallback.py b/src/vfbquery/term_info_fallback.py new file mode 100644 index 0000000..197172d --- /dev/null +++ b/src/vfbquery/term_info_fallback.py @@ -0,0 +1,379 @@ +"""Build a missing ``term_info`` SOLR document on demand. + +``get_term_info`` reads a pre-built ``term_info`` document out of the +``vfb_json`` SOLR collection. Those documents are written in bulk by +`VFB_json_schema_indexer `_, +driven by the ``precompute live query results`` Jenkins job. A record that +reaches the PDB after the last successful run of that job therefore has no +document at all, and every VFBquery call for it returns ``None`` until the +job next completes — which is not a matter of hours: build #71 started on +2026-09-04, #70 (2026-09-02) was aborted, and the last success before that +was #66 on 2026-06-22. + +``VFB_00107fob`` ("ME_R on JRC2018Unisex") is the case that surfaced this. +It is a perfectly good painted domain — its ``in_register_with`` edge, its +images and its parent class are all in the PDB — but with no SOLR document +it is invisible to term info, and it silently vanished from the medulla +class page as well. + +This module closes that window. On a miss we run *the indexer's own query* +for the node's type, hand the row to the same deserialiser ``get_term_info`` +already uses, and write the result back so the next caller gets it from +SOLR. Nothing here re-implements the schema: the queries, the document +shape and the SOLR write are all imported from the indexer, which is cloned +into the image by the Dockerfile exactly as the Jenkins job clones it. If a +term_info query changes upstream, this follows it on the next image build. + +The write is deliberately conservative: + +* only when the id genuinely has no document — an existing one is never + overwritten, and the indexer's own atomic-update shape (``{"term_info": + {"set": ...}}``) means the sibling precomputed fields on the document + (``anat_query``, ``anat_2_ep_query``, ``ep_2_anat_query``) are untouched + either way; +* never when the SOLR cache is disabled, so a test run against live data + cannot write into the shared production collection. That has bitten VFB + before: a fixture written into the production cache namespace poisoned + JRC2018U term info for five days. + +If the indexer is not importable — a plain ``pip install vfbquery`` has no +reason to carry it — every entry point here degrades to "no fallback" and +``get_term_info`` behaves exactly as it did before. +""" + +import contextlib +import json +import os +import sys +import threading + +# -------------------------------------------------------------------------- +# Optional import of the indexer. Present in the Docker image (see Dockerfile, +# which clones VFB_json_schema_indexer and copies VFB_json_schema into its +# src/vfb, mirroring the Jenkins job); absent from a plain pip install. +# -------------------------------------------------------------------------- + +_IMPORT_ERROR = None +_INDEXERS = None +_send_solr_docs = None +_import_lock = threading.Lock() + + +def _pin_schema_version(): + """Stop ``query_roller.get_version_tag`` shelling out to git. + + It runs ``git rev-parse --short HEAD`` against the *process* working + directory on every query build and raises ``CalledProcessError`` when + that is not a git repository — which it never is for a running + VFBquery container. The value only lands in the document's ``version`` + field, so serve the SHA the image was built from instead. + """ + try: + from src.vfb.vfb_query_builder import query_roller + except ImportError: + return + query_roller.get_version_tag = lambda: schema_version() + + +#: Written by the Dockerfile after it resolves the two clones, so a rebuilt +#: document records which schema produced it even when the build tracked a +#: branch rather than a pinned SHA. +VERSIONS_FILE = os.getenv("VFB_VERSIONS_FILE", "/opt/vfb_versions.env") + + +def schema_version(): + """The VFB_json_schema commit this image was built from. + + Environment first (so a deployment can override), then the file the + Dockerfile writes, then a marker that is obviously not a SHA. + """ + from_env = os.getenv("VFB_JSON_SCHEMA_SHA") + if from_env: + return from_env + try: + with open(VERSIONS_FILE) as f: + for line in f: + key, _, value = line.strip().partition("=") + if key == "VFB_JSON_SCHEMA_SHA" and value: + return value + except OSError: + pass + return "unpinned" + + +def _seed_indexer_env(): + """Point the indexer's Neo4j connection at the one VFBquery already uses. + + ``BaseQueryIndexer.__init__`` reads PDBserver/PDBuser/PDBpassword from the + environment and builds its own connection. Rather than run the fallback + against a different database than the rest of the process, fill those in + from VFBquery's own client when the deployment has not set them. Anything + already in the environment wins. + """ + from .vfb_queries import vc + nc = vc.nc + defaults = { + "PDBserver": getattr(nc, "base_uri", None), + "PDBuser": getattr(nc, "usr", None), + "PDBpassword": getattr(nc, "pwd", None), + } + for key, value in defaults.items(): + if value and not os.getenv(key): + os.environ[key] = value + + +#: Where the Dockerfile puts the indexer checkout. +INDEXER_ROOT = os.getenv("VFB_INDEXER_ROOT", "/opt/vfb_indexer") + + +@contextlib.contextmanager +def _indexer_importable(): + """Make ``src`` mean the indexer's package for the duration of an import. + + The indexer's top-level package is literally called ``src``, and so is + VFBquery's own source directory -- which carries an ``__init__.py`` + (``from vfbquery import *``) and so is a *regular* package. Whichever + comes first on ``sys.path`` wins outright and the running process always + finds its own first, so ``import src.indexers`` fails with + ModuleNotFoundError even with the indexer on PYTHONPATH. This is not + hypothetical: it is what the first build of this module did. + + Put the indexer root first and evict any ``src`` already imported, then + put both back. Safe because nothing in VFBquery imports ``src`` by name, + and because every ``from src...`` in the indexer is module-level: they + all resolve while this context is open, so restoring afterwards cannot + strand a later lookup. + """ + saved = {name: mod for name, mod in sys.modules.items() + if name == "src" or name.startswith("src.")} + for name in saved: + del sys.modules[name] + sys.path.insert(0, INDEXER_ROOT) + try: + yield + finally: + try: + sys.path.remove(INDEXER_ROOT) + except ValueError: + pass + for name in [n for n in sys.modules + if n == "src" or n.startswith("src.")]: + del sys.modules[name] + sys.modules.update(saved) + + +def _load_indexers(): + """Import the indexer classes once, or record why we could not.""" + global _INDEXERS, _send_solr_docs, _IMPORT_ERROR + if _INDEXERS is not None or _IMPORT_ERROR is not None: + return + with _import_lock: + if _INDEXERS is not None or _IMPORT_ERROR is not None: + return + try: + _seed_indexer_env() + with _indexer_importable(): + _pin_schema_version() + from src.indexers.term_info.anatomical_ind_term_info_indexer import ( + AnatomicalIndTermInfoQueryIndexer) + from src.indexers.term_info.class_term_info_indexer import ( + ClassTermInfoQueryIndexer) + from src.indexers.term_info.cluster_term_info_indexer import ( + ClusterTermInfoQueryIndexer) + from src.indexers.term_info.dataset_term_info_indexer import ( + DatasetTermInfoQueryIndexer) + from src.indexers.term_info.license_term_info_indexer import ( + LicenseTermInfoQueryIndexer) + from src.indexers.term_info.neuron_class_term_info_indexer import ( + NeuronClassTermInfoQueryIndexer) + from src.indexers.term_info.pub_term_info_indexer import ( + PubTermInfoQueryIndexer) + from src.indexers.term_info.split_class_term_info_indexer import ( + SplitClassTermInfoQueryIndexer) + from src.indexers.term_info.template_term_info_indexer import ( + TemplateTermInfoQueryIndexer) + from src.solr_client import send_solr_docs + except Exception as e: # ImportError, or a missing env var + _IMPORT_ERROR = e + return + _INDEXERS = { + "template": TemplateTermInfoQueryIndexer, + "license": LicenseTermInfoQueryIndexer, + "dataset": DatasetTermInfoQueryIndexer, + "pub": PubTermInfoQueryIndexer, + "cluster": ClusterTermInfoQueryIndexer, + "anatomical_ind": AnatomicalIndTermInfoQueryIndexer, + "neuron_class": NeuronClassTermInfoQueryIndexer, + "split_class": SplitClassTermInfoQueryIndexer, + "class": ClassTermInfoQueryIndexer, + } + _send_solr_docs = send_solr_docs + + +def fallback_available(): + """True when the indexer imported and a document can be built.""" + _load_indexers() + return _INDEXERS is not None + + +_warned_unavailable = False + + +def _warn_unavailable_once(): + """Say once why no document can be built, so an operator can tell + "the indexer is not in this image" from "the build was attempted and + failed". Once, not per request: a container without the indexer would + otherwise log this on every miss.""" + global _warned_unavailable + if _warned_unavailable: + return + _warned_unavailable = True + print("term_info fallback unavailable, missing documents will not be " + "rebuilt (%s)" % fallback_unavailable_reason()) + + +def fallback_unavailable_reason(): + """Why :func:`fallback_available` is False, for logging. None if it is True.""" + _load_indexers() + if _INDEXERS is not None: + return None + return "%s: %s" % (type(_IMPORT_ERROR).__name__, _IMPORT_ERROR) + + +# -------------------------------------------------------------------------- +# Type dispatch +# -------------------------------------------------------------------------- + +#: Ids the indexer's parameter queries exclude, so we exclude them too rather +#: than writing a document the bulk job would never have written. +EXCLUDED_ID_PREFIXES = ("VFBc_", "FBlc", "SAMN", "VFB_internal") + + +def choose_indexer(labels): + """Pick the indexer whose population this node belongs to. + + Mirrors, per node, the ``get_parameters_query`` predicates that each + indexer applies across the whole graph. Order matters: the anatomical + individual query is the one that excludes all the other Individual + types, so it is tried after them. + + :param labels: the node's neo4j labels + :return: key into the indexer table, or None if no indexer covers it + """ + labels = set(labels or ()) + if "Template" in labels: + return "template" + if "License" in labels: + return "license" + if "DataSet" in labels: + return "dataset" + if "pub" in labels and "Individual" in labels: + return "pub" + if "Cluster" in labels and "Individual" in labels: + return "cluster" + if "Individual" in labels: + return "anatomical_ind" + if "Class" in labels: + if "Neuron" in labels: + return "neuron_class" + if "Split" in labels: + return "split_class" + return "class" + return None + + +def _node_labels(short_form, neo): + """Labels for one node, or None if the PDB does not have it either.""" + from .vfb_queries import get_dict_cursor + rows = get_dict_cursor()(neo.commit_list([ + "MATCH (n) WHERE n.short_form = '%s' RETURN labels(n) AS labels" + % short_form.replace("'", "\\'") + ])) + if not rows: + return None + return rows[0].get("labels") or [] + + +# -------------------------------------------------------------------------- +# Build / write +# -------------------------------------------------------------------------- + +def build_term_info(short_form, neo=None): + """Build one term's ``term_info`` payload live from the PDB. + + :param short_form: the term's short_form + :param neo: a Neo4jConnect; defaults to VFBquery's own connection + :return: ``(payload_json, solr_doc)``, or ``(None, None)`` + """ + if not fallback_available(): + _warn_unavailable_once() + return None, None + if short_form.startswith(EXCLUDED_ID_PREFIXES): + print("term_info fallback: %s is excluded from the term_info index" + % short_form) + return None, None + + from .vfb_queries import vc, get_dict_cursor + neo = neo or vc.nc + + labels = _node_labels(short_form, neo) + if labels is None: + print("term_info fallback: %s is not in the PDB either" % short_form) + return None, None + key = choose_indexer(labels) + if key is None: + print("term_info fallback: no term_info indexer covers labels %s (%s)" + % (sorted(labels), short_form)) + return None, None + + indexer = _INDEXERS[key]() + rows = get_dict_cursor()(neo.commit_list([ + indexer.get_vfb_json_query([short_form])])) + if not rows: + print("term_info fallback: %s query returned no rows for %s" + % (key, short_form)) + return None, None + + result = rows[0] + # The indexer's own document shape, including its atomic-update wrapper. + solr_doc = indexer.generate_solr_doc(result, request=[short_form]) + return json.dumps(result), solr_doc + + +def write_term_info(solr_doc): + """Write one document with the indexer's retry-hardened SOLR client. + + :return: True when SOLR accepted it + """ + if not fallback_available(): + return False + from .solr_result_cache import solr_caching_disabled + if solr_caching_disabled(): + print("term_info fallback: cache disabled, not writing %s to SOLR" + % solr_doc.get("id")) + return False + try: + return bool(_send_solr_docs([solr_doc], "term_info")) + except Exception as e: + print("term_info fallback: SOLR write failed for %s: %s" + % (solr_doc.get("id"), e)) + return False + + +def backfill_term_info(short_form): + """Build a missing document, write it back, and return the payload. + + Called by ``get_term_info`` when SOLR has no document for the id. The + caller has already established the miss, so the write cannot overwrite + anything. + + :return: the ``term_info`` payload as a JSON string, or None + """ + payload, solr_doc = build_term_info(short_form) + if payload is None: + return None + if write_term_info(solr_doc): + print("term_info fallback: built and indexed %s" % short_form) + else: + print("term_info fallback: built %s but did not index it" % short_form) + return payload diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 347bcb9..9fd2aa8 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1,5 +1,6 @@ import pysolr from .term_info_queries import deserialize_term_info +from .term_info_fallback import backfill_term_info # Replace VfbConnect import with our new SimpleVFBConnect from .owlery_client import SimpleVFBConnect # Keep dict_cursor if it's used elsewhere - lazy import to avoid GUI issues @@ -2528,6 +2529,21 @@ def FindComboPublications_to_schema(name, take_default): ) +class _FallbackSolrResult: + """A one-document stand-in for a pysolr result. + + ``term_info_parse_object`` only ever reads ``hits`` and + ``docs[0]['term_info'][0]``, so a document rebuilt by + :mod:`vfbquery.term_info_fallback` can be fed straight back through the + same parser rather than round-tripping through SOLR — which matters + because the write is best-effort and may legitimately be skipped. + """ + + def __init__(self, term_info_payload): + self.hits = 1 + self.docs = [{"term_info": [term_info_payload]}] + + # term_info SOLR loaders: fetch one term's term_info doc by short_form and # return it either as a deserialized object (attribute access) or as the raw # JSON dict, whichever the caller works with. @@ -2831,6 +2847,22 @@ def get_term_info(short_form: str, preview: bool = True, force_refresh: bool = F try: # Search for the term in the SOLR server results = vfb_solr.search('id:' + short_form) + # SOLR has no term_info document for this id. That is routine for a + # record newer than the last successful run of the bulk indexer (the + # `precompute live query results` Jenkins job), which can be months + # behind: the term is fine in the PDB, it just has nothing to read. + # Build the document live from the indexer's own query and index it, + # so this id is only ever slow once. + # + # Tested on the SOLR result rather than on the parse: a miss does not + # give a falsy parse. term_info_parse_object skips its whole body when + # there are no hits and returns the initialised skeleton, which then + # fails schema validation on Name/Id/Meta and is returned raw -- so a + # missing term used to surface as a truthy object with no Id, not None. + if not getattr(results, "hits", 0): + fallback_payload = backfill_term_info(short_form) + if fallback_payload: + results = _FallbackSolrResult(fallback_payload) # Check if any results were returned parsed_object = term_info_parse_object(results, short_form) if parsed_object: