diff --git a/Dockerfile b/Dockerfile index 2b13f3d..26196ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,18 +25,22 @@ RUN pip install --no-cache-dir --no-binary numpy \ # copies the schema's src into src/vfb and puts the checkout on PYTHONPATH. # Do exactly that, so what runs here is what runs in the bulk job. # -# Pin by passing --build-arg INDEXER_REF=; the default tracks master the -# way the Jenkins job does. Both resolved SHAs are baked into the image so -# /status can report which schema built a given document. -ARG INDEXER_REF=master -ARG JSON_SCHEMA_REF=master +# Pin by passing --build-arg INDEXER_REF=; the default, `default`, keeps +# whatever branch the clone checks out, the way the Jenkins job does. Do not +# name a branch here: the two repositories disagree (the indexer's default is +# `main`, the schema's is `master`), and hardcoding either breaks the other -- +# and would break again silently if a default branch were ever renamed. Both +# resolved SHAs are baked into the image so a rebuilt document can say which +# schema produced it. +ARG INDEXER_REF=default +ARG JSON_SCHEMA_REF=default RUN apt-get update && \ apt-get install -y --no-install-recommends git && \ rm -rf /var/lib/apt/lists/* RUN git clone --quiet https://github.com/VirtualFlyBrain/VFB_json_schema_indexer.git /opt/vfb_indexer && \ - git -C /opt/vfb_indexer checkout --quiet "${INDEXER_REF}" && \ + { [ "${INDEXER_REF}" = default ] || git -C /opt/vfb_indexer checkout --quiet "${INDEXER_REF}"; } && \ git clone --quiet https://github.com/VirtualFlyBrain/VFB_json_schema.git /opt/vfb_json_schema && \ - git -C /opt/vfb_json_schema checkout --quiet "${JSON_SCHEMA_REF}" && \ + { [ "${JSON_SCHEMA_REF}" = default ] || git -C /opt/vfb_json_schema checkout --quiet "${JSON_SCHEMA_REF}"; } && \ mkdir -p /opt/vfb_indexer/src/vfb && \ cp -r /opt/vfb_json_schema/src/* /opt/vfb_indexer/src/vfb/ && \ printf 'VFB_INDEXER_SHA=%s\nVFB_JSON_SCHEMA_SHA=%s\n' \ diff --git a/src/test/test_term_info_fallback.py b/src/test/test_term_info_fallback.py index 7814598..6c3b974 100644 --- a/src/test/test_term_info_fallback.py +++ b/src/test/test_term_info_fallback.py @@ -52,15 +52,16 @@ def test_template_wins_over_individual(): # --------------------------------------------------------------------------- -# Exclusions -- ids the bulk indexer never writes a document for +# Exclusions -- anatomical-branch only # --------------------------------------------------------------------------- -@pytest.mark.parametrize("short_form", [ - "VFBc_00000001", "FBlc0006125", "SAMN12345678", "VFB_internal_thing", -]) -def test_excluded_ids_are_not_built(short_form): - payload, doc = tif.build_term_info(short_form) - assert payload is None and doc is None +def test_exclusions_are_scoped_to_the_anatomical_branch(): + """FBlc ids are Clusters, which have their own indexer. The anatomical + indexer's parameter query excludes them; the term_info index as a whole + does not. Applying the list globally would refuse every Cluster.""" + assert "FBlc" in tif.ANATOMICAL_EXCLUDED_ID_PREFIXES + assert tif.choose_indexer(["Individual", "Cluster"]) == "cluster" + assert tif.choose_indexer(["Individual", "Anatomy"]) == "anatomical_ind" # --------------------------------------------------------------------------- diff --git a/src/vfbquery/term_info_fallback.py b/src/vfbquery/term_info_fallback.py index 197172d..a001609 100644 --- a/src/vfbquery/term_info_fallback.py +++ b/src/vfbquery/term_info_fallback.py @@ -244,9 +244,12 @@ def fallback_unavailable_reason(): # Type dispatch # -------------------------------------------------------------------------- -#: Ids the indexer's parameter queries exclude, so we exclude them too rather -#: than writing a document the bulk job would never have written. -EXCLUDED_ID_PREFIXES = ("VFBc_", "FBlc", "SAMN", "VFB_internal") +#: Id prefixes the *anatomical individual* indexer's parameter query excludes. +#: They are not excluded from the term_info index as a whole: FBlc ids are +#: Clusters, which ClusterTermInfoQueryIndexer owns, so this list only applies +#: on the anatomical branch. Applying it globally -- as the first version of +#: this module did -- would have refused to rebuild any Cluster document. +ANATOMICAL_EXCLUDED_ID_PREFIXES = ("VFBc_", "FBlc", "SAMN", "VFB_internal") def choose_indexer(labels): @@ -308,10 +311,6 @@ def build_term_info(short_form, neo=None): if not fallback_available(): _warn_unavailable_once() return None, None - if short_form.startswith(EXCLUDED_ID_PREFIXES): - print("term_info fallback: %s is excluded from the term_info index" - % short_form) - return None, None from .vfb_queries import vc, get_dict_cursor neo = neo or vc.nc @@ -325,6 +324,13 @@ def build_term_info(short_form, neo=None): print("term_info fallback: no term_info indexer covers labels %s (%s)" % (sorted(labels), short_form)) return None, None + if (key == "anatomical_ind" + and short_form.startswith(ANATOMICAL_EXCLUDED_ID_PREFIXES)): + # The anatomical indexer's parameter query skips these, so the bulk + # job would never have written a document for one either. + print("term_info fallback: %s is excluded from the anatomical " + "individual index" % short_form) + return None, None indexer = _INDEXERS[key]() rows = get_dict_cursor()(neo.commit_list([