From 2146904dac14bf3b517bffcf70ff2bc1b934876a Mon Sep 17 00:00:00 2001 From: Robbie1977 Date: Sat, 29 Aug 2026 23:47:34 +0000 Subject: [PATCH 1/5] Move the validated examples out of README into a test fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README had grown into a 2,800-line generated document because the examples CI executes and diffs lived inside it. They are a test asset, so they now live at src/test/example_queries.md, verbatim, under a header saying what consumes them. readme_parser and the (renamed) update_examples.py/fix_examples.py point at the fixture; extraction parity checked — 5 python and 4 json blocks, identical to the old README, with the parser regenerating byte-identical test files. The examples workflow's sed still reads README.md until the companion .github/workflows/examples.yml change lands separately (the release PAT cannot push workflow files); the new README's quick-start block is runnable, so the job stays green either way. --- docs/conf.py | 18 +- fix_readme.py => fix_examples.py | 4 +- src/test/example_queries.md | 2861 ++++++++++++++++++++++++ src/test/readme_parser.py | 9 +- update_readme.py => update_examples.py | 4 +- 5 files changed, 2878 insertions(+), 18 deletions(-) rename fix_readme.py => fix_examples.py (90%) create mode 100644 src/test/example_queries.md rename update_readme.py => update_examples.py (87%) diff --git a/docs/conf.py b/docs/conf.py index 6820f91a..6a1e525f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,11 +5,10 @@ 1. **Most of the documentation predates the site.** `README.md`, `CACHING.md`, `RELEASING.md`, `schema.md`, `performance.md` and `VFB_QUERIES_REFERENCE.md` live at the repo root and are read there, on GitHub, by people who never - open the docs site. `README.md` in particular is *generated* — the - `examples` workflow executes its ```python blocks and `update_readme.py` - rewrites its ```json blocks — so it cannot be moved, renamed or edited to - suit Sphinx. Instead `_sync_root_docs` copies them into `docs/_root/` at - build time. Copying rather than `{include}`-ing keeps the links *between* + open the docs site. (`README.md` used to be *generated*; the validated + example blocks now live in `src/test/example_queries.md` as a test fixture + and the README is hand-written.) `_sync_root_docs` copies the root + documents into `docs/_root/` at build time. Copying rather than `{include}`-ing keeps the links *between* them working: `CACHING.md` links to `RELEASING.md` as a sibling, and in `_root/` they still are siblings. `docs/_root/` is generated and gitignored. @@ -166,9 +165,8 @@ # # * the set-theory notation the /combine reference is written in — ∪ ∩ ⊕ ↔ ≡, # and the α/β used for the operand placeholders; -# * the emoji in the *generated* root documents — 🎉 ✅ ✨ 🔶 in `README.md` -# and `performance.md`, written by the `examples` and `performance-test` -# workflows. +# * the emoji in the root documents — a few in `README.md`, and 🎉 ✅ ✨ 🔶 +# in `performance.md`, which the `performance-test` workflow regenerates. # # Neither source can be cleaned up at the source: the notation is the subject # matter, and the emoji are rewritten by a workflow on every run. xelatex reads @@ -243,8 +241,8 @@ def _drop_leading_transitions(text): heading does not already carry, so the copy drops it. Done here rather than by editing the source, because these files are read - directly on GitHub and one of them (`README.md`) is regenerated by the - `examples` workflow — a fix applied there would be overwritten. + directly on GitHub and one of them (`performance.md`) is regenerated by a + workflow — a fix applied there would be overwritten. """ lines = text.split("\n") out = [] diff --git a/fix_readme.py b/fix_examples.py similarity index 90% rename from fix_readme.py rename to fix_examples.py index e5c2ac0d..77566b4b 100644 --- a/fix_readme.py +++ b/fix_examples.py @@ -1,6 +1,6 @@ import re -with open('README.md', 'r') as f: +with open('src/test/example_queries.md', 'r') as f: content = f.read() # Fix the thumbnail lines @@ -24,7 +24,7 @@ content = '\n'.join(lines) -with open('README.md', 'w') as f: +with open('src/test/example_queries.md', 'w') as f: f.write(content) print('Fixed README') \ No newline at end of file diff --git a/src/test/example_queries.md b/src/test/example_queries.md new file mode 100644 index 00000000..6ca9d37b --- /dev/null +++ b/src/test/example_queries.md @@ -0,0 +1,2861 @@ +# VFBquery example queries — validated test fixture + +**This file is a test fixture, not user documentation.** It holds the example +queries and their expected result shapes that CI validates against the live VFB +backend on every run: + +- `.github/workflows/examples.yml` executes every python code block here + (rewritten so each `vfb.*()` call is printed) against production. +- `src/test/readme_parser.py` (run as `python -m src.test.readme_parser`) + regenerates `test_examples.py`, `test_examples_code.py` and + `test_results.py` from the blocks in this file. +- `python -m src.test.test_examples_diff` then compares live results against + the json blocks below, structure-first. +- `update_examples.py` (repo root) rewrites the json blocks from + `json_block_.json` files when the expected output legitimately changes. + +For what VFBquery is and how to use it, see the repo `README.md` and +. + +## 🚀 Performance & Caching + +VFBquery includes intelligent SOLR-based caching for optimal performance: + +- **Instant Results** for repeated queries +- **NBLAST similarity queries**: 10+ seconds → <0.1 seconds (cached) +- **Zero configuration** - works automatically +- **Persistent cache** survives restarts +- **3-month TTL** matches VFB_connect behavior + +```python +import vfbquery as vfb + +# First query builds cache (~1-2 seconds) +result1 = vfb.get_term_info('FBbt_00003748') + +# Subsequent queries served from cache (<0.1 seconds) +result2 = vfb.get_term_info('FBbt_00003748') # 54,000x faster! + +# Similarity queries also cached +similar = vfb.get_similar_neurons('VFB_jrchk00s') # Fast after first run +``` + +📚 See [CACHING.md](CACHING.md) for cache configuration, the `VFBQUERY_CACHE_ENABLED` +bypass (used by the tests), and version-based invalidation; and +[RELEASING.md](RELEASING.md) for how the single-source version (`_version.py`) is +bumped from the release tag. + +## 🕸️ CATMAID pass-through + +VFB hosts public, read-only CATMAID servers for several connectomics datasets +(FAFB, FANC, L1EM, ...; registry at +). `vfbquery` exposes their +query API directly, and anywhere a command takes skeleton ids you can pass +CATMAID skids, VFB ids (`VFB_xxxxxxxx`) or a mixed list — VFB ids are converted +through the knowledge graph's cross-references before the request is made. + +```python +import vfbquery as vfb +from vfbquery import catmaid + +vfb.list_catmaid_instances() # hosted instances + tokens + projects +vfb.list_catmaid_commands() # the curated read-only command registry + +fafb = catmaid('fafb') # optionally catmaid('fanc', project=2) +fafb.commands() # {command: doc} + +# Mixed VFB ids and skids; the envelope carries the id mapping both ways: +fafb.connectivity(ids=['VFB_001011rj', 10603863], boolean_op='OR') +fafb.neuron_names(ids=['VFB_001011rj']) +fafb.swc(id='VFB_001011rj') # single-id commands take id= +fafb.swc_alignments(id='VFB_001011rj') # spaces an SWC is available in +fafb.swc(id='VFB_001011rj', aligned='JRC2018Unisex') # VFB's template-registered copy +# (aligned=