From 8a7bb4f2287ef89ba4c94a55a0acd574772a1ec3 Mon Sep 17 00:00:00 2001 From: Robbie Court Date: Fri, 4 Sep 2026 07:21:28 +0000 Subject: [PATCH] Fix the image build, and stop the fallback refusing Clusters Two defects in the term_info fallback as merged. The Docker build failed. It checked both clones out at "master", but the two repositories disagree: VFB_json_schema_indexer's default branch is `main` and VFB_json_schema's is `master`, so `git checkout master` in the indexer failed with "pathspec 'master' did not match any file(s)". The refs now default to the sentinel `default`, which skips the checkout and keeps whatever the clone gave us; an explicit --build-arg still pins as before. Naming a branch here would break again, silently, if either default were renamed. (The first failure on this hunk was different and is now moot: VFB_json_schema_indexer was private, so the anonymous clone could not authenticate. It is public as of today.) EXCLUDED_ID_PREFIXES was applied to every id, but that list comes from the *anatomical individual* indexer's parameter query. FBlc ids are Clusters, which ClusterTermInfoQueryIndexer owns, so as written the fallback refused to rebuild any Cluster document -- the one term type whose query exists only in the indexer and not in QueryLibrary, and so the one this feature is least able to do without. The list is renamed ANATOMICAL_EXCLUDED_ID_PREFIXES and applied only on that branch. Verified live: FBlc0006125 now rebuilds ("scRNAseq_2018_Davie_FULL_seq_clustering_adPN_neurons", types include Cluster), and the clone/checkout chain was run through sh with both a default and an explicit ref. --- Dockerfile | 18 +++++++++++------- src/test/test_term_info_fallback.py | 15 ++++++++------- src/vfbquery/term_info_fallback.py | 20 +++++++++++++------- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2b13f3d..26196ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,18 +25,22 @@ RUN pip install --no-cache-dir --no-binary numpy \ # copies the schema's src into src/vfb and puts the checkout on PYTHONPATH. # Do exactly that, so what runs here is what runs in the bulk job. # -# Pin by passing --build-arg INDEXER_REF=; the default tracks master the -# way the Jenkins job does. Both resolved SHAs are baked into the image so -# /status can report which schema built a given document. -ARG INDEXER_REF=master -ARG JSON_SCHEMA_REF=master +# Pin by passing --build-arg INDEXER_REF=; the default, `default`, keeps +# whatever branch the clone checks out, the way the Jenkins job does. Do not +# name a branch here: the two repositories disagree (the indexer's default is +# `main`, the schema's is `master`), and hardcoding either breaks the other -- +# and would break again silently if a default branch were ever renamed. Both +# resolved SHAs are baked into the image so a rebuilt document can say which +# schema produced it. +ARG INDEXER_REF=default +ARG JSON_SCHEMA_REF=default RUN apt-get update && \ apt-get install -y --no-install-recommends git && \ rm -rf /var/lib/apt/lists/* RUN git clone --quiet https://github.com/VirtualFlyBrain/VFB_json_schema_indexer.git /opt/vfb_indexer && \ - git -C /opt/vfb_indexer checkout --quiet "${INDEXER_REF}" && \ + { [ "${INDEXER_REF}" = default ] || git -C /opt/vfb_indexer checkout --quiet "${INDEXER_REF}"; } && \ git clone --quiet https://github.com/VirtualFlyBrain/VFB_json_schema.git /opt/vfb_json_schema && \ - git -C /opt/vfb_json_schema checkout --quiet "${JSON_SCHEMA_REF}" && \ + { [ "${JSON_SCHEMA_REF}" = default ] || git -C /opt/vfb_json_schema checkout --quiet "${JSON_SCHEMA_REF}"; } && \ mkdir -p /opt/vfb_indexer/src/vfb && \ cp -r /opt/vfb_json_schema/src/* /opt/vfb_indexer/src/vfb/ && \ printf 'VFB_INDEXER_SHA=%s\nVFB_JSON_SCHEMA_SHA=%s\n' \ diff --git a/src/test/test_term_info_fallback.py b/src/test/test_term_info_fallback.py index 7814598..6c3b974 100644 --- a/src/test/test_term_info_fallback.py +++ b/src/test/test_term_info_fallback.py @@ -52,15 +52,16 @@ def test_template_wins_over_individual(): # --------------------------------------------------------------------------- -# Exclusions -- ids the bulk indexer never writes a document for +# Exclusions -- anatomical-branch only # --------------------------------------------------------------------------- -@pytest.mark.parametrize("short_form", [ - "VFBc_00000001", "FBlc0006125", "SAMN12345678", "VFB_internal_thing", -]) -def test_excluded_ids_are_not_built(short_form): - payload, doc = tif.build_term_info(short_form) - assert payload is None and doc is None +def test_exclusions_are_scoped_to_the_anatomical_branch(): + """FBlc ids are Clusters, which have their own indexer. The anatomical + indexer's parameter query excludes them; the term_info index as a whole + does not. Applying the list globally would refuse every Cluster.""" + assert "FBlc" in tif.ANATOMICAL_EXCLUDED_ID_PREFIXES + assert tif.choose_indexer(["Individual", "Cluster"]) == "cluster" + assert tif.choose_indexer(["Individual", "Anatomy"]) == "anatomical_ind" # --------------------------------------------------------------------------- diff --git a/src/vfbquery/term_info_fallback.py b/src/vfbquery/term_info_fallback.py index 197172d..a001609 100644 --- a/src/vfbquery/term_info_fallback.py +++ b/src/vfbquery/term_info_fallback.py @@ -244,9 +244,12 @@ def fallback_unavailable_reason(): # Type dispatch # -------------------------------------------------------------------------- -#: Ids the indexer's parameter queries exclude, so we exclude them too rather -#: than writing a document the bulk job would never have written. -EXCLUDED_ID_PREFIXES = ("VFBc_", "FBlc", "SAMN", "VFB_internal") +#: Id prefixes the *anatomical individual* indexer's parameter query excludes. +#: They are not excluded from the term_info index as a whole: FBlc ids are +#: Clusters, which ClusterTermInfoQueryIndexer owns, so this list only applies +#: on the anatomical branch. Applying it globally -- as the first version of +#: this module did -- would have refused to rebuild any Cluster document. +ANATOMICAL_EXCLUDED_ID_PREFIXES = ("VFBc_", "FBlc", "SAMN", "VFB_internal") def choose_indexer(labels): @@ -308,10 +311,6 @@ def build_term_info(short_form, neo=None): if not fallback_available(): _warn_unavailable_once() return None, None - if short_form.startswith(EXCLUDED_ID_PREFIXES): - print("term_info fallback: %s is excluded from the term_info index" - % short_form) - return None, None from .vfb_queries import vc, get_dict_cursor neo = neo or vc.nc @@ -325,6 +324,13 @@ def build_term_info(short_form, neo=None): print("term_info fallback: no term_info indexer covers labels %s (%s)" % (sorted(labels), short_form)) return None, None + if (key == "anatomical_ind" + and short_form.startswith(ANATOMICAL_EXCLUDED_ID_PREFIXES)): + # The anatomical indexer's parameter query skips these, so the bulk + # job would never have written a document for one either. + print("term_info fallback: %s is excluded from the anatomical " + "individual index" % short_form) + return None, None indexer = _INDEXERS[key]() rows = get_dict_cursor()(neo.commit_list([