From 2cddfaf8591b41cb6b9b9906e47760ed657f0a48 Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 6 Jul 2026 14:22:38 -0700 Subject: [PATCH 01/13] Create secret-scan.yml for Gitleaks integration Add GitHub Actions workflow for secret scanning using Gitleaks. --- .github/workflows/secret-scan.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/secret-scan.yml diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml new file mode 100644 index 0000000..33df52e --- /dev/null +++ b/.github/workflows/secret-scan.yml @@ -0,0 +1,14 @@ +name: Secret Scan + +on: + pull_request: + push: + +jobs: + gitleaks: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: gitleaks/gitleaks-action@v2 From e20ec9cde85996212eeb87a2c388ad7b2d3b0508 Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 6 Jul 2026 14:28:03 -0700 Subject: [PATCH 02/13] Add GITHUB_TOKEN --- .github/workflows/secret-scan.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index 33df52e..8c145ee 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -11,4 +11,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: gitleaks/gitleaks-action@v2 + - name: Gitleaks + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e93e26df76b9b35632c9dcf2597d9884680726eb Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 6 Jul 2026 14:31:37 -0700 Subject: [PATCH 03/13] Add workflow_dispatch trigger to secret scan --- .github/workflows/secret-scan.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index 8c145ee..6b5cd3e 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -3,15 +3,22 @@ name: Secret Scan on: pull_request: push: + workflow_dispatch: jobs: - gitleaks: + secret-scan: runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Gitleaks - uses: gitleaks/gitleaks-action@v2 + - name: Run Gitleaks + uses: gitleaks/gitleaks-action@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c292c4ab33e0374d53181561bb607317bc6afedd Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 6 Jul 2026 14:38:38 -0700 Subject: [PATCH 04/13] Add pinned commit SHA --- .github/workflows/secret-scan.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index 6b5cd3e..9992e56 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -5,20 +5,21 @@ on: push: workflow_dispatch: +permissions: + contents: read + security-events: write + jobs: - secret-scan: + gitleaks: runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Run Gitleaks - uses: gitleaks/gitleaks-action@v2 + uses: gitleaks/gitleaks-action@2cddfaf8591b41cb6b9b9906e47760ed657f0a48 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From b317896ab9f7156f3525e21be767cff8dc19dfb6 Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 6 Jul 2026 14:40:20 -0700 Subject: [PATCH 05/13] Update Gitleaks action to use version v2 --- .github/workflows/secret-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index 9992e56..b7a7f93 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -20,6 +20,6 @@ jobs: fetch-depth: 0 - name: Run Gitleaks - uses: gitleaks/gitleaks-action@2cddfaf8591b41cb6b9b9906e47760ed657f0a48 + uses: gitleaks/gitleaks-action@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 035a74522041830560fd092d09d3695592dcd827 Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 13 Jul 2026 10:24:29 -0700 Subject: [PATCH 06/13] Add SECURITY.md for compliance --- .github/workflows/secret-scan.yml | 25 ------------------ SECURITY.md | 43 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 25 deletions(-) delete mode 100644 .github/workflows/secret-scan.yml create mode 100644 SECURITY.md diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml deleted file mode 100644 index b7a7f93..0000000 --- a/.github/workflows/secret-scan.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Secret Scan - -on: - pull_request: - push: - workflow_dispatch: - -permissions: - contents: read - security-events: write - -jobs: - gitleaks: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Run Gitleaks - uses: gitleaks/gitleaks-action@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..30d76de --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,43 @@ +# Security Policy + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations. + +## Reporting Security Vulnerabilities + +**Please do not report security vulnerabilities through public GitHub issues, discussions, or pull requests.** + +Instead, report them to the Microsoft Security Response Center (MSRC): + +- Report a vulnerability: https://msrc.microsoft.com/create-report +- MSRC vulnerability reporting guidance: https://www.microsoft.com/msrc + +If you prefer to submit without logging in, use: + +- https://www.microsoft.com/msrc/report-a-vulnerability + +You should receive a response within 24 hours. If you do not receive a response, please follow up via the reporting portal. + +Please include as much information as possible to help us reproduce and investigate the issue: + +- Type of issue +- Full paths of affected files or components +- Steps to reproduce +- Proof-of-concept code (if available) +- Potential impact assessment + +## Supported Versions + +As this project is under active development, security fixes are typically provided in the latest version of the repository. + +Users are encouraged to: +- Use the latest released version. +- Keep dependencies up to date. +- Follow Azure and Microsoft security best practices when deploying solutions based on this repository. + +## Additional Resources + +For more information about Microsoft's vulnerability disclosure process, see: + +- Microsoft Security Response Center: https://www.microsoft.com/msrc +- Coordinated Vulnerability Disclosure: https://www.microsoft.com/msrc/cv +d From 4e05a9afeaead77c885e82cbcc6856f39099766e Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 13 Jul 2026 10:27:44 -0700 Subject: [PATCH 07/13] Fix link for Coordinated Vulnerability Disclosure --- SECURITY.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 30d76de..8b20f7e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -39,5 +39,4 @@ Users are encouraged to: For more information about Microsoft's vulnerability disclosure process, see: - Microsoft Security Response Center: https://www.microsoft.com/msrc -- Coordinated Vulnerability Disclosure: https://www.microsoft.com/msrc/cv -d +- Coordinated Vulnerability Disclosure: https://www.microsoft.com/msrc/cvd From b77a904d34cd07a61517c9d8a907a9dd588611b2 Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 3 Aug 2026 18:09:58 +0000 Subject: [PATCH 08/13] Onboard the Query Performance Tuning article as agent skills Onboards https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/ into the kit: - New standalone skill `documentdb-query-performance-tuning` (end-to-end tuning methodology + `references/documentdb-explain-output.md` for DocumentDB's Postgres-backed explain format). - Merge the former `documentdb-query-optimization` rule into `documentdb-query-optimizer` (explain-first ordering + confounders); content preserved as `references/query-explain-plan.md`. - Extend the knowledge-base router to route to skills (Route B): `skills[]` + `routes_one_hop_skills` in kb.json, `rank_skills` + `--skills` in the engine; scripts routing unchanged. Regression-guarded (kb-router scenario). - Authoritative `VCoreMongoRequests` slow-query KQL in the monitoring rule. - Reproducible before/after test on the ecommerce dataset (`scenarios/ecommerce/query-perf-skill-test.{js,sh}`), an engineering write-up (`docs/query-perf-skill-test-instruction-*.md`), and a customer-facing quickstart (`docs/quickstart-find-and-fix-slow-queries.md`). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 86722878-7b4c-4d57-933c-3d4188317703 --- .github/agents/skill-reviewer/rubric.md | 2 +- AGENTS.md | 8 +- README.md | 7 +- docs/SKILLS.md | 11 +- ...f-skill-test-instruction-20260730180441.md | 322 ++++++++++++++++++ docs/quickstart-find-and-fix-slow-queries.md | 193 +++++++++++ knowledge-base/README.md | 85 +++-- knowledge-base/kb-route.sh | 20 +- knowledge-base/kb.json | 140 +++++++- knowledge-base/kb_route.py | 136 ++++++-- scenarios/ecommerce/README.md | 7 + scenarios/ecommerce/query-perf-skill-test.js | 157 +++++++++ scenarios/ecommerce/query-perf-skill-test.sh | 41 +++ .../data-modeling/model-large-field-split.md | 2 +- .../monitoring/monitoring-slow-query-log.md | 27 +- skills/query-optimization/SKILL.md | 13 - .../query-optimization/query-explain-plan.md | 30 -- skills/query-optimizer/SKILL.md | 78 ++++- .../references/query-explain-plan.md | 58 ++++ skills/query-performance-tuning/SKILL.md | 218 ++++++++++++ .../references/documentdb-explain-output.md | 127 +++++++ skills/sharding/SKILL.md | 2 +- .../sharding-hot-partition-diagnosis.md | 2 +- skills/sharding/sharding-when-to-shard.md | 2 +- .../kb-router/expected-findings.yaml | 41 +++ .../kb-router/tests/test_kb_router.py | 71 ++++ 26 files changed, 1667 insertions(+), 133 deletions(-) create mode 100644 docs/query-perf-skill-test-instruction-20260730180441.md create mode 100644 docs/quickstart-find-and-fix-slow-queries.md create mode 100644 scenarios/ecommerce/query-perf-skill-test.js create mode 100755 scenarios/ecommerce/query-perf-skill-test.sh delete mode 100644 skills/query-optimization/SKILL.md delete mode 100644 skills/query-optimization/query-explain-plan.md create mode 100644 skills/query-optimizer/references/query-explain-plan.md create mode 100644 skills/query-performance-tuning/SKILL.md create mode 100644 skills/query-performance-tuning/references/documentdb-explain-output.md diff --git a/.github/agents/skill-reviewer/rubric.md b/.github/agents/skill-reviewer/rubric.md index 3685fe5..c86451e 100644 --- a/.github/agents/skill-reviewer/rubric.md +++ b/.github/agents/skill-reviewer/rubric.md @@ -13,7 +13,7 @@ The single highest-leverage check. The frontmatter `description` is loaded every Grade against three questions: 1. **Triggers.** Does it list at least 2–3 concrete situations the user might be in? ("Use when designing a new schema, migrating from SQL, deciding between embedding and referencing…") A description that says only *what* the skill is, not *when* to invoke it, fails this check. -2. **Differentiation.** Could the description plausibly match another skill in the same kit? If `documentdb-indexing` and `documentdb-query-optimization` could trade descriptions and nothing would change, both descriptions are too generic. +2. **Differentiation.** Could the description plausibly match another skill in the same kit? If `documentdb-indexing` and `documentdb-query-optimizer` could trade descriptions and nothing would change, both descriptions are too generic. 3. **Length.** Anthropic's hard limit is 1024 characters. Practical sweet spot is 200–500 characters: long enough to list real triggers, short enough that the routing model reads it cleanly. < 80 chars is almost always too vague. > 800 chars usually means the body leaked into the description. | Grade | Rule | diff --git a/AGENTS.md b/AGENTS.md index 277023b..4605d0f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,7 +58,6 @@ These skills each describe a feature area and link to short rule files with inco | Skill | Folder | When to use | |---|---|---| | `documentdb-data-modeling` | `skills/data-modeling/` | Designing schemas, embed vs reference, 16 MB limit, denormalization, schema versioning | -| `documentdb-query-optimization` | `skills/query-optimization/` | Writing queries that use indexes; reading `explain("executionStats")` | | `documentdb-indexing` | `skills/indexing/` | Choosing the right index type (single / compound / multikey / wildcard / hashed / 2dsphere / TTL); ESR ordering; query-pattern → index-shape cookbook; safe index lifecycle (`hideIndex` → `dropIndex`) | | `documentdb-driver` | `skills/driver/` | Singleton `MongoClient`, connection reuse fundamentals | | `documentdb-vector-search` | `skills/vector-search/` | `cosmosSearch` with DiskANN / HNSW / IVF, PQ, fp16, cosine normalization | @@ -78,16 +77,21 @@ These skills walk the user (or another agent) through a task end-to-end. | `documentdb-azure-deployment` | `skills/azure-deployment/` | Provisioning an Azure DocumentDB cluster (`Microsoft.DocumentDB/mongoClusters`) via Bicep, Azure CLI, Terraform, or portal; firewall rules; connection string retrieval | | `documentdb-natural-language-querying` | `skills/natural-language-querying/` | "How do I query…", "filter / group / aggregate…", SQL → MQL translation (read-only queries only) | | `documentdb-query-optimizer` | `skills/query-optimizer/` | "Why is this slow?", index review, `explain()`-driven tuning; loads `references/core-indexing-principles.md` | +| `documentdb-query-performance-tuning` | `skills/query-performance-tuning/` | End-to-end tuning methodology: reading DocumentDB's Postgres-backed `explain("executionStats")`, the ESR rule, index-backed sorts, covered queries, finding slow queries via Log Analytics `VCoreMongoRequests`; loads `references/documentdb-explain-output.md` | | `documentdb-connection` | `skills/connection/` | Pool-size / timeout / retry tuning for serverless, OLTP, OLAP, or bursty workloads | ## Routing hints for agents These map a task to the best **Route B (text) skill**. (On **Route A** — when the user said *"use toolbox"* — route the same task through `knowledge-base/kb-route.sh` -to a diagnostic script instead.) +to a diagnostic script instead.) `kb-route.sh` can also deterministically surface +the best-matching **skill** for these Route-B tasks (`--skills`, or the +`skill_match` / `recommended` fields in `--json` output) if you want a scored pick +rather than reading the table below. - **Writing / generating a query** → `documentdb-natural-language-querying` - **"Why is this query slow / how do I index this?"** → `documentdb-query-optimizer` +- **"How do I read explain output / what is the ESR rule / how do I tune query performance / find slow queries in prod"** → `documentdb-query-performance-tuning` - **"Which index type should I use / design this index"** → `documentdb-indexing` - **Designing a schema / data model** → `documentdb-data-modeling` - **Adding vector search to a RAG app** → `documentdb-vector-search` diff --git a/README.md b/README.md index bd1f2e1..e178950 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ Skills follow the [Agent Skills](https://agentskills.io/) format and the kit shi 👉 **Capabilities and skill catalog:** [`docs/SKILLS.md`](docs/SKILLS.md) +👉 **New here? 10-minute quick start:** [Find & fix a slow query](docs/quickstart-find-and-fix-slow-queries.md) — run DocumentDB locally, load sample data, and let your AI assistant diagnose a `COLLSCAN` and fix it with one index (50,000 docs scanned → 10). + ## Diagnostic Toolbox — Quickstart Beyond the text skills, the kit ships **deterministic diagnostic scripts** and a @@ -60,6 +62,9 @@ bash scripts/index-redundancy-finder.sh --db ecommerce # 3. or ask in natural language — the router picks the tool (no LLM, no container) bash knowledge-base/kb-route.sh --db contoso "why are my aggregations slow even though I have indexes" + +# the same router also picks the best text skill for guidance questions +bash knowledge-base/kb-route.sh --skills # or: kb-route.sh "how do I read explain output" ``` Demo datasets are seeders under [`scenarios/`](scenarios/) (they plant the @@ -80,7 +85,7 @@ skills/ SKILL.md # agent-facing activation + instructions references/ # reference docs the skill loads at runtime scripts/ # diagnostic toolbox — read-only analyzers + seeders -knowledge-base/ # NL → script router (kb.json + kb_route.py) + demo +knowledge-base/ # NL → script + skill router (kb.json + kb_route.py) + demo scenarios/contoso/ # ready-to-run TOAST demo dataset (+ optional scaling-benchmark/) testing/ # fixture-first regression suite for the scripts (pytest) token-tests/ # measured token savings of scripts vs text-skill workflows diff --git a/docs/SKILLS.md b/docs/SKILLS.md index a657508..cbcd786 100644 --- a/docs/SKILLS.md +++ b/docs/SKILLS.md @@ -19,7 +19,6 @@ why it matters → incorrect example → correct example → references. |---|---|---| | [`data-modeling/`](../skills/data-modeling/) | `model-` | Embed vs reference, 16 MB limit, denormalization, schema versioning | | [`sharding/`](../skills/sharding/) | `sharding-` | When to shard vs stay single-shard, shard-key selection (read-heavy vs write-heavy), logical/physical shard mental model, scale-up vs scale-out, hot-partition diagnosis, `sh.shardCollection` / `sh.reshardCollection`, 4 TB logical-shard budget | -| [`query-optimization/`](../skills/query-optimization/) | `query-` | `explain("executionStats")`, avoiding `COLLSCAN` | | [`indexing/`](../skills/indexing/) | `index-` | Index-type selection (single / compound-ESR / multikey / wildcard / hashed / 2dsphere / TTL), query-pattern → index-shape cookbook, index budget, safe `hideIndex` → `dropIndex` lifecycle | | [`driver/`](../skills/driver/) | `driver-` | MongoDB driver/SDK usage (singleton client, pooling) | | [`vector-search/`](../skills/vector-search/) | `vector-` | `cosmosSearch` with DiskANN / HNSW / IVF, PQ, fp16 | @@ -39,7 +38,8 @@ Single-purpose skills the agent loads when its trigger description matches. | [`mcp-setup/`](../skills/mcp-setup/) | Configuring the DocumentDB MCP server (connection string, transport, shell profile) | | [`azure-deployment/`](../skills/azure-deployment/) | Provisioning an Azure DocumentDB cluster (`Microsoft.DocumentDB/mongoClusters`) — Bicep (with Key Vault), Azure CLI one-shot, Terraform pointer, firewall, connection string, teardown. See also [`examples/azure-deployment/`](../examples/azure-deployment/) for a no-agent ready-to-run deploy. | | [`natural-language-querying/`](../skills/natural-language-querying/) | "How do I query…", filter/aggregate/group requests, SQL → MQL translation | -| [`query-optimizer/`](../skills/query-optimizer/) | "Why is this query slow?", index review, `explain()`-driven tuning (indexing deep-dive lives in its `references/`) | +| [`query-optimizer/`](../skills/query-optimizer/) | "Why is this query slow?", index review, `explain()`-driven tuning, and **verifying** a query uses an index vs `COLLSCAN` (absorbs the former `query-optimization` rule); indexing deep-dive + explain-verification live in its `references/` | +| [`query-performance-tuning/`](../skills/query-performance-tuning/) | End-to-end tuning methodology: read DocumentDB's Postgres-backed `explain("executionStats")`, the ESR rule, index-backed sorts, covered queries, and finding slow queries via Log Analytics `VCoreMongoRequests` (explain field glossary lives in its `references/`) | | [`connection/`](../skills/connection/) | Connection pool / timeout / retry tuning; serverless vs OLTP vs OLAP patterns | ## Diagnostic toolbox (scripts + router) @@ -47,7 +47,10 @@ Single-purpose skills the agent loads when its trigger description matches. Beyond the text skills, the kit ships **deterministic, read-only diagnostic scripts** that inspect a *local* DocumentDB container across both layers (MongoDB API + PostgreSQL engine), plus a **knowledge-base router** that maps a natural- -language question to the exact script — no LLM at routing time. Full guide: +language question to the exact target — no LLM at routing time. The router scores +a question against **two spaces**: Route A `tools` (the scripts below) and Route B +`skills` (the standalone skills above), and reports the best of each plus a +`recommended` route. Full guide: [`DIAGNOSTICS.md`](DIAGNOSTICS.md); catalog: [`../README.md`](../README.md#the-tools-scripts). | Tool | Answers | @@ -57,7 +60,7 @@ language question to the exact script — no LLM at routing time. Full guide: | [`db-config-advisor.sh`](../scripts/db-config-advisor.sh) | Working set vs cache, TOAST share, cache-hit ratios (evidence-based). | | [`perf-advisor.sh`](../scripts/perf-advisor.sh) | Overall health: collection-scan audit, query timing, PG I/O / locks / config. | | [`data-integrity-check.sh`](../scripts/data-integrity-check.sh) | Orphaned foreign keys + mixed-type fields (hard structural integrity). | -| [`knowledge-base/`](../knowledge-base/README.md) | NL question → exact script (deterministic keyword scoring, zero deps). | +| [`knowledge-base/`](../knowledge-base/README.md) | NL question → exact script (`--list`) **or** skill (`--skills`); deterministic keyword scoring, zero deps. | Companion to the toolbox: the `data-modeling` skill's [`model-large-field-split`](../skills/data-modeling/model-large-field-split.md) diff --git a/docs/query-perf-skill-test-instruction-20260730180441.md b/docs/query-perf-skill-test-instruction-20260730180441.md new file mode 100644 index 0000000..c1dccd3 --- /dev/null +++ b/docs/query-perf-skill-test-instruction-20260730180441.md @@ -0,0 +1,322 @@ +# Query-Performance Skill Test — Instruction & Results + +> **Generated:** 2026-07-30 18:04:41 UTC · **Re-run (post-merge):** 2026-07-30 21:18 UTC — identical scan-work results; test 4.1 re-labeled to `documentdb-query-optimizer` after the `query-optimization` merge · **Status:** Public Preview (see repo `AGENTS.md`) +> **Article under test:** [Query Performance Tuning Guide](https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/) (Azure DocumentDB devblog) +> **Skills exercised:** `documentdb-query-performance-tuning`, `documentdb-query-optimizer` (two facets: index recommendation + explain-plan verification, the latter merged from the former `documentdb-query-optimization`), `documentdb-indexing` (`index-compound-esr`) + +This document is a **reproducible before/after test** that validates the query-performance +skills against a **real local DocumentDB container**. It takes a realistic slow query per case, +captures `explain("executionStats")` **before** any supporting index (the `COLLSCAN` baseline), +applies the index that skill recommends, and re-captures explain **after**. Every number below is +measured, not estimated. The 2026-07-30 21:18 UTC re-run (after `query-optimization` was merged +into `query-optimizer`) reproduced every scan-work figure exactly. + +**Reproducible test artifacts** (added with this doc): +- `scenarios/ecommerce/query-perf-skill-test.js` — the mongosh measurement harness +- `scenarios/ecommerce/query-perf-skill-test.sh` — the container runner + +> ### ⚠️ Key caveat (read first) +> The **headline optimization reproduces** here: every skill turns a 50,000-doc `COLLSCAN` +> into a selective `IXSCAN`. But two of the article's *further* optimizations — **index-backed +> sort** (removing the in-memory `SORT`) and **covered queries** (removing the `FETCH`) — **do +> not reproduce on the `documentdb-local` image**, because the experimental engine flags that +> enable them ship **off** and are **pinned per-session by the gateway** (so even `ALTER SYSTEM` +> can't flip them). These are managed **Azure DocumentDB** behaviors, per the article. Full, +> verified detail and the fuller list of gated flags is in [§6 Caveats](#6-honest-caveats-local-image-vs-managed-azure-documentdb). + +--- + +## 1. Is the `scenarios/` dataset a good fit? — Yes: `ecommerce` + +The existing **`ecommerce`** seed (`scenarios/ecommerce/seed.sh`) is an excellent match for the +article. Its `orders` collection is seeded with **50,000 documents and *no* secondary index** +(only the default `_id`), so the "before" state is a genuine full collection scan — exactly the +article's starting point. The document shape maps almost 1:1 onto the article's example: + +| Article field | `ecommerce.orders` field | Notes | +|---|---|---| +| `status: "shipped"` | `status` ∈ {pending, confirmed, shipped, delivered, cancelled} | `"shipped"` ≈ 10,127 / 50,000 docs (low cardinality) | +| `customerId: "C-4821"` | `customer_id` = `CUST_000000`…`CUST_004999` | 5,000 distinct → **high cardinality** (~10 orders each), like the article's customerId | +| `createdAt` | `created_at` (all in 2024) | supports the `$gte: ISODate("2024-01-01")` range + sort | +| (order total) | `total_amount` | used for the range-predicate (ESR) case | +| (city) | `shipping_city` ∈ 10 cities | used for the two-equality case | + +The only differences from the article are **field naming** (`customer_id`/`created_at` vs +`customerId`/`createdAt`) and **scale** (50K docs vs the article's 1M). The before/after story +is identical. + +> The other datasets are **not** a fit for this article: `contoso` targets large-text/TOAST +> bloat and `index-redundancy` targets duplicate/unused indexes — different diagnostics. + +--- + +## 2. Environment & prerequisites + +| Item | Value | +|---|---| +| Container image | `ghcr.io/microsoft/documentdb/documentdb-local@sha256:0fcf634531c1917ad0855ff9f4354aca0a5c5d8e435f08250568deb37eeb0ad5` | +| Container name | `documentdb-local` | +| Gateway (Mongo API) | container-internal `localhost:10260`, TLS, SCRAM-SHA-256 | +| PostgreSQL engine | container-internal `localhost:9712` | +| Credentials | user `docdbadmin`, password `Test1234` (`DB_PASSWORD`) | +| mongosh client | `2.3.8` | + +> **Reproducibility gotcha:** this `documentdb-local` image **does not bundle a `mongosh` +> client**, but `scenarios/*/seed.sh` and the harness invoke `mongosh` via `docker exec`. Install +> it into the container once (root): +> ```bash +> docker exec -u root documentdb-local bash -lc ' +> cd /tmp && wget -q https://downloads.mongodb.com/compass/mongosh-2.3.8-linux-x64.tgz -O m.tgz && +> tar xzf m.tgz && cp mongosh-2.3.8-linux-x64/bin/mongosh /usr/local/bin/ && +> cp mongosh-2.3.8-linux-x64/bin/mongosh_crypt_v1.so /usr/local/lib/ 2>/dev/null; mongosh --version' +> ``` + +### Bring up + seed + +```bash +# 1. start the container (host ports optional; seeders use docker exec) +docker run -d --name documentdb-local -e USERNAME=docdbadmin -e PASSWORD=Test1234 \ + ghcr.io/microsoft/documentdb/documentdb-local:latest +# 2. install mongosh into the container (see gotcha above) +# 3. seed the ecommerce dataset (~50K orders; takes ~1 min) +export DB_PASSWORD=Test1234 +bash scenarios/ecommerce/seed.sh +``` + +--- + +## 3. How each skill is triggered (exact query words) + +All trigger phrases resolve **deterministically** in the router's skill space (Route B). + +> **Post-run update:** since this test was generated, the former `documentdb-query-optimization` +> rule skill was **merged into `documentdb-query-optimizer`** (its explain-verification content now +> lives in `query-optimizer/references/query-explain-plan.md`). So test 4.1 below is now the +> *explain-plan verification* facet of `query-optimizer`, and its trigger routes to `query-optimizer`. + +| Skill | Type | Exact trigger phrase | `kb-route.sh` result (verified) | +|---|---|---|---| +| `documentdb-query-optimizer` (explain-plan verification) | standalone | "use explain to check whether this query uses an index or does a collection scan" | → skill **`query-optimizer`** (7.10) ✅ | +| `documentdb-query-optimizer` | standalone | "optimize this query and recommend an index" | → skill **`query-optimizer`** (9.00) ✅ | +| `documentdb-query-performance-tuning` | standalone | "how do I read the explain output and apply the ESR rule to tune this query" | → skill **`query-performance-tuning`** (11.38) ✅ | +| `documentdb-indexing` (`index-compound-esr`) | rule-folder | "which compound index should I design for this filter, sort, and range query" | → skill **`indexing`** (7.50) ✅ | + +Try any phrase yourself: +```bash +bash knowledge-base/kb-route.sh --json "optimize this query and recommend an index" +``` + +--- + +## 4. Before/after results (measured) + +**How to read these.** Azure DocumentDB's planner is PostgreSQL-backed, so the **top-line** +`totalDocsExamined` is a *post-sort* count and can look tiny even during a full scan — the +article warns about exactly this. The honest "work" metric is **documents/keys examined at the +scan stage**, which the harness drills down to. `executionTimeMillis` is the **min of 5 runs** +but is noisy at this data size (see caveats). + +Each test runs against `ecommerce.orders` (50,000 docs). "Scan work" = docs examined by a +`COLLSCAN` vs keys examined by the `IXSCAN`. + +### 4.1 `documentdb-query-optimizer` (explain-plan verification, merged from `query-optimization`) + +**Goal:** explain() literacy — a `COLLSCAN` becomes an `IXSCAN` once an index supports the filter. + +```javascript +db.orders.find({ order_id: "ORD_012345" }) // single high-selectivity equality +db.orders.createIndex({ order_id: 1 }) // the fix +``` + +| Metric | Before | After | +|---|---|---| +| Scan stage | **COLLSCAN** | **IXSCAN** (`order_id_1`) → FETCH | +| Examined at scan | **50,000 docs** | **1 key** | +| Blocking SORT | – | – | +| nReturned | 1 | 1 | +| `executionTimeMillis` (min/5) | 9 ms | 0 ms | +| Top-line `totalDocsExamined` | 50,000 | 1 | + +**Scan work: 50,000 → 1 (≈50,000× fewer).** The cleanest case: no sort, so the plan collapses to +a bare index lookup + fetch. + +### 4.2 `documentdb-query-optimizer` — recommend a compound index + +**Goal:** recommend a compound index for a specific slow query (two equality fields + sort). + +```javascript +db.orders.find({ status: "shipped", shipping_city: "Seattle" }).sort({ created_at: -1 }) +db.orders.createIndex({ status: 1, shipping_city: 1, created_at: -1 }) // ESR: 2 equality, then sort +``` + +| Metric | Before | After | +|---|---|---| +| Scan stage | **COLLSCAN** | **IXSCAN** (`status_1_shipping_city_1_created_at_-1`) → FETCH | +| Examined at scan | **50,000 docs** | **998 keys** | +| Blocking SORT | yes | yes¹ | +| nReturned | 998 | 998 | +| `executionTimeMillis` (min/5) | 14 ms | 3 ms | + +**Scan work: 50,000 → 998 (≈50× fewer).** ¹The blocking `SORT` persists on this local image — see +caveat §6.1. + +### 4.3 `documentdb-query-performance-tuning` — the article's flagship query (ESR + covered) + +**Goal:** full ESR compound index with the **most-selective equality first**, plus the +covered-query variant. + +```javascript +db.orders.find({ + status: "shipped", + customer_id: "CUST_004087", + created_at: { $gte: ISODate("2024-01-01") } +}).sort({ created_at: -1 }) +db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 }) // customer_id first = most selective +// covered variant adds projection { _id:0, customer_id:1, status:1, created_at:1 } +``` + +| Metric | Before | After | After (covered) | +|---|---|---|---| +| Scan stage | **COLLSCAN** | **IXSCAN** → FETCH | **IXSCAN** → FETCH² | +| Examined at scan | **50,000 docs** | **10 keys** | **10 keys** | +| Blocking SORT | yes | yes¹ | yes¹ | +| nReturned | 10 | 10 | 10 | +| `executionTimeMillis` (min/5) | 12 ms | 28 ms³ | 28 ms³ | + +**Scan work: 50,000 → 10 (≈5,000× fewer).** Putting the high-cardinality `customer_id` first is +why this drops to 10 keys while §4.2 (leading with low-cardinality `status`) only drops to 998 — +a direct demonstration of the article's *"most selective field first"* rule. +²The covered projection does **not** remove the `FETCH` on this local image (caveat §6.1). +³Wall-clock is noisy on a 10-row result (caveat §6.2); the scan-work reduction is the real signal. + +### 4.4 `documentdb-indexing` — `index-compound-esr` (Range LAST) + +**Goal:** ESR with a range predicate — Equality, then Sort, then **Range last**. + +```javascript +db.orders.find({ status: "shipped", total_amount: { $gt: 500 } }).sort({ created_at: -1 }) +db.orders.createIndex({ status: 1, created_at: -1, total_amount: 1 }) // E → S → R +``` + +| Metric | Before | After | +|---|---|---| +| Scan stage | **COLLSCAN** | **IXSCAN** (`status_1_created_at_-1_total_amount_1`) → FETCH | +| Examined at scan | **50,000 docs** | **8,899 keys** | +| Blocking SORT | yes | yes¹ | +| nReturned | 8,899 | 8,899 | +| `executionTimeMillis` (min/5) | 24 ms | 42 ms³ | + +**Scan work: 50,000 → 8,899 (≈5.6× fewer).** The modest reduction is expected and instructive: +`status: "shipped"` + `total_amount > 500` is a **low-selectivity** predicate (most shipped +orders exceed \$500), so even a correct ESR index cannot narrow much. Selectivity, not index +correctness alone, bounds the win. + +--- + +## 5. Summary + +| Skill | Query shape | Recommended index | Scan work (before → after) | Reduction | +|---|---|---|---|---| +| `query-optimizer` (explain verification) | 1 equality, no sort | `{order_id:1}` | 50,000 → 1 | **≈50,000×** | +| `query-optimizer` | 2 equality + sort | `{status:1, shipping_city:1, created_at:-1}` | 50,000 → 998 | **≈50×** | +| `query-performance-tuning` | 2 equality + range + sort | `{customer_id:1, status:1, created_at:-1}` | 50,000 → 10 | **≈5,000×** | +| `indexing` | 1 equality + range + sort | `{status:1, created_at:-1, total_amount:1}` | 50,000 → 8,899 | **≈5.6×** | + +**Every skill turns a 50,000-document `COLLSCAN` into an `IXSCAN`.** The magnitude of the win +tracks **selectivity** exactly as the article predicts: high-cardinality leading equality fields +(`order_id`, `customer_id`) yield 1000×–50000× reductions; low-cardinality predicates +(`status`, coarse ranges) yield single- to double-digit reductions. + +--- + +## 6. Honest caveats (local image vs managed Azure DocumentDB) + +### 6.1 Index-backed sort and covered queries are gated OFF on `documentdb-local` + +The article's *further* optimizations — **eliminating the in-memory `SORT`** (index-backed sort) +and **eliminating the `FETCH`** (covered query) — **did not reproduce** on this local image. In +every "after" plan with a sort, a blocking `SORT` stage remained, and the covered-query variant +(§4.3) kept its `FETCH`. This is a **local-image configuration limitation, not a skill error.** + +**What I verified (measured):** +1. With the defaults, the flagship "after" plan is `SORT → FETCH → IXSCAN` and the covered + variant is byte-for-byte identical to the non-covered one (`FETCH` never drops). +2. I enabled the full experimental combo at the engine level — + `enableNewCompositeIndexOpClass=on`, `defaultUseCompositeOpClass=on`, + `enableIndexOrderbyPushdown=on`, `forceRumIndexScantoBitmapHeapScan=off` — via + `ALTER SYSTEM … ; SELECT pg_reload_conf();`, confirmed each shows `on` in a **fresh psql + session**, then **dropped and recreated** the index so it would be built under the new op + class. The Mongo-API `explain` **still** returned `SORT → FETCH → IXSCAN`. +3. Conclusion: **the gateway pins these settings per-session** (see its startup + *"Dynamic configurations loaded"* log), so PostgreSQL-level `ALTER SYSTEM` does not change the + Mongo-API query path. On the local image these experimental flags are simply off. (I reset all + four GUCs back to their defaults afterward; the harness itself never touches engine config.) + +**The relevant flags gated OFF on `documentdb-local`** (names/descriptions are the engine's own — +several literally say *"new experimental"*). The article demonstrates the corresponding behavior on +managed **Azure DocumentDB**, implying they are enabled there. + +| Engine flag | Local default | What it governs | Relevance to the article | Evidence | +|---|---|---|---|---| +| `documentdb.enableIndexOrderbyPushdown` | `off` | pushes `ORDER BY` into the composite index | **index-backed sort** (removes the blocking `SORT`) | ✅ measured: `SORT` persists | +| `documentdb.enableNewCompositeIndexOpClass` | `off` | the "new experimental composite index opclass" | prerequisite for ordered **and** covering composite indexes | ✅ measured: combo-on + index rebuild still `SORT`/`FETCH` | +| `documentdb.defaultUseCompositeOpClass` | `off` | whether default `createIndex` builds use that op class | so the recommended index is ordered/covering | ⬤ config-observed | +| `documentdb.enableSortbyIdPushDownToPrimaryKey` | `off` | pushes `sort({_id})` onto the primary key | index-backed sort on `_id` | ⬤ config-observed | +| `documentdb.forceRumIndexScantoBitmapHeapScan` | `on` | forces a bitmap heap scan (always re-fetches heap tuples) | **prevents index-only / covered scans** → `FETCH` always present | ✅ measured: covered variant keeps `FETCH` | +| `documentdb.enableNewSelectivityMode` | `off` | the "new selectivity logic" in the planner | the article's explain surfaces `indexCosts` / `selectivity` / `correlation` | ⬤ config-observed | +| `documentdb.enableMultiIndexRumJoin` | `off` | intersecting multiple indexes for one query | combining indexes vs one compound index | ⬤ config-observed | +| `documentdb.enablePrimaryKeyCursorScan` | `off` | primary-key cursor scan for streaming cursors | faster `_id`-ordered cursor scans | ⬤ config-observed | + +✅ = behavior measured in this test · ⬤ = observed in engine config (`pg_settings`), not separately behavior-tested here. + +**Beyond query tuning**, the same image also ships these adjacent performance features off (not +exercised by this test, listed for completeness): `documentdb.enable_force_push_vector_index=off` +and `documentdb.enableVectorPreFilterV2=off` (vector-search push-down / pre-filter v2). HNSW, +product-quantization and half-precision vector compression **are** on. + +> **How to see the full set yourself:** +> ```bash +> docker exec documentdb-local psql -h localhost -p 9712 -U documentdb -d postgres \ +> -c "SELECT name, setting, short_desc FROM pg_settings WHERE name LIKE 'documentdb.%' ORDER BY name;" +> ``` + +### 6.2 `executionTimeMillis` is noisy at this scale + +On small result sets some "after" timings exceed "before" (e.g. §4.3, 12 ms → 28 ms) due to plan +caching, index-page warmup, and sub-millisecond measurement noise. This is itself one of the +article's lessons — *time can look fine while the query does far too much work* — so this test +reports **documents/keys examined at the scan stage** as the stable, honest signal. + +### 6.3 Scale + +50,000 docs vs the article's 1,000,000. The plan **shapes** and scan-work **ratios** are what +matter and are scale-independent; absolute milliseconds are not comparable. + +--- + +## 7. Reproduce this test + +```bash +# prerequisites: container up + mongosh installed + ecommerce seeded (see §2) +export DB_PASSWORD=Test1234 +bash scenarios/ecommerce/query-perf-skill-test.sh # prints the human report + a JSON block +``` + +The harness (`scenarios/ecommerce/query-perf-skill-test.js`) is self-contained: for each skill it +drops every non-`_id` index on `orders` (forcing the baseline), measures before, creates the +recommended index, measures after (+ the covered variant for the flagship), and drops the index — +so it is **idempotent** and leaves `orders` with only its default `_id` index. + +--- + +## 8. Verdict + +- **Dataset fit:** `ecommerce.orders` is a strong, faithful stand-in for the article's `orders` + example — no new dataset needed. +- **Skills validated:** all four test cases (across three skills — `query-optimizer` covers two + facets) turn a `COLLSCAN` into an `IXSCAN`, with the reduction scaling by selectivity exactly as + the ESR guidance predicts. The recommended indexes and trigger phrases are the ones the skills + ship. +- **Faithful limitation:** index-backed-sort and covered-query removal are environment-gated on + the local image; they are Azure-DocumentDB-managed behaviors (per the article), not something + the local test can demonstrate. diff --git a/docs/quickstart-find-and-fix-slow-queries.md b/docs/quickstart-find-and-fix-slow-queries.md new file mode 100644 index 0000000..32503b9 --- /dev/null +++ b/docs/quickstart-find-and-fix-slow-queries.md @@ -0,0 +1,193 @@ +# Quick Start: Find & Fix a Slow Query on Azure DocumentDB + +**Time: ~10 minutes.** In this guide you'll run Azure DocumentDB locally, load a +sample store dataset, ask your AI coding assistant *why a query is slow*, apply the +one-line fix it recommends, and watch the same query go from scanning **50,000 +documents** to just **10** — with no application code change. + +You don't need to know anything about the internals. If you have Docker and an AI +coding assistant, you can follow along by copy-paste. + +--- + +## What is this? + +The **Azure DocumentDB Agent Kit** is a set of *skills* you add to your AI coding +assistant (Claude Code, Cursor, GitHub Copilot CLI, Gemini CLI, VS Code, …). Once +installed, your assistant knows how to diagnose and optimize +[Azure DocumentDB](https://learn.microsoft.com/azure/documentdb/) (the managed, +MongoDB-compatible database) — reading `explain()` output, designing indexes, and +more. + +Install the skills (one time): + +```bash +npx skills add Azure/documentdb-agent-kit +``` + +For the sample data and the ready-made test used below, also grab the repo: + +```bash +git clone https://github.com/Azure/documentdb-agent-kit +cd documentdb-agent-kit +``` + +> **Preview:** Azure DocumentDB and this kit are in public preview; details may +> change. No production SLA. + +--- + +## Step 1 — Run DocumentDB locally + +```bash +docker run -d --name documentdb-local \ + -e USERNAME=docdbadmin -e PASSWORD=Test1234 \ + ghcr.io/microsoft/documentdb/documentdb-local:latest +``` + +Add a Mongo shell to the container (the preview image ships without one, and the +sample-data loader uses it): + +```bash +docker exec -u root documentdb-local bash -lc ' + cd /tmp && wget -q https://downloads.mongodb.com/compass/mongosh-2.3.8-linux-x64.tgz -O m.tgz && + tar xzf m.tgz && cp mongosh-2.3.8-linux-x64/bin/mongosh /usr/local/bin/ && + cp mongosh-2.3.8-linux-x64/bin/mongosh_crypt_v1.so /usr/local/lib/ 2>/dev/null; mongosh --version' +``` + +## Step 2 — Load the sample data + +This seeds a realistic store database (`ecommerce`) — **50,000 orders**, +customers, products, and more. It takes about a minute. + +```bash +export DB_PASSWORD=Test1234 +bash scenarios/ecommerce/seed.sh +``` + +The `orders` collection starts with **no indexes** (other than the default `_id`) +— exactly the situation that makes queries slow as data grows. + +--- + +## Step 3 — Ask your assistant why a query is slow + +Here's a perfectly reasonable query — find a customer's shipped orders since the +start of the year, newest first: + +```javascript +db.orders.find({ + status: "shipped", + customer_id: "CUST_004087", + created_at: { $gte: ISODate("2024-01-01") } +}).sort({ created_at: -1 }) +``` + +Paste this prompt to your AI assistant (the kit's skill will pick it up): + +> **"Why is this query slow, and how do I fix it?"** +> `db.orders.find({ status: "shipped", customer_id: "CUST_004087", created_at: { $gte: ISODate("2024-01-01") } }).sort({ created_at: -1 })` + +Your assistant will run `explain("executionStats")` and spot the problem: a +**collection scan** (`COLLSCAN`) — the database reads **every one** of the 50,000 +documents to return just 10. + +> **Prefer to look yourself?** Run the same query with `.explain("executionStats")`. +> ⚠️ **Read the scan stage, not the summary.** The top-line time can look tiny (a +> few milliseconds) even during a full scan — look at the `COLLSCAN` stage, where +> `totalDocsExamined` is **50,000**. + +*(No assistant handy? The kit also ships a deterministic router that names the +right skill for a question — `bash knowledge-base/kb-route.sh "why is this query slow and how do I index it"`.)* + +--- + +## Step 4 — Apply the recommended fix + +The skill recommends a single **compound index**, ordered by the **ESR rule** +(Equality → Sort → Range), with the most selective field first: + +```javascript +db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 }) +``` + +Create it (your assistant will ask for approval first): + +```bash +docker exec -u documentdb documentdb-local mongosh "localhost:10260/ecommerce" \ + -u docdbadmin -p Test1234 --authenticationMechanism SCRAM-SHA-256 --tls \ + --tlsAllowInvalidCertificates --quiet \ + --eval 'db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 })' +``` + +## Step 5 — See the improvement + +Run `explain("executionStats")` on the same query again. The `COLLSCAN` is gone, +replaced by an **index scan** (`IXSCAN`) that examines only the rows it needs: + +| | Before (no index) | After (ESR index) | +|---|---|---| +| Plan | `COLLSCAN` (full scan) | `IXSCAN` (index scan) | +| Documents/keys examined | **50,000** | **10** | +| Rows returned | 10 | 10 | + +**~5,000× less work per query.** Same result, a fraction of the resources — which +means more headroom to scale without adding infrastructure. + +--- + +## Try a few more (one command) + +The kit includes a ready-made test that runs several *before/after* comparisons +for you and prints the results: + +```bash +export DB_PASSWORD=Test1234 +bash scenarios/ecommerce/query-perf-skill-test.sh +``` + +You'll see the same pattern across different query shapes — the size of the win +depends on how *selective* the filter is: + +| Ask your assistant… | Recommended index | Examined: before → after | +|---|---|---| +| "Does this query use an index or a collection scan?" (`find` by `order_id`) | `{ order_id: 1 }` | 50,000 → **1** | +| "Optimize `find({status, shipping_city}).sort({created_at})`" | `{ status: 1, shipping_city: 1, created_at: -1 }` | 50,000 → **998** | +| "Tune the flagship `find({status, customer_id, created_at}).sort()`" | `{ customer_id: 1, status: 1, created_at: -1 }` | 50,000 → **10** | +| "Which compound index for `find({status, total_amount:{$gt}}).sort({created_at})`?" | `{ status: 1, created_at: -1, total_amount: 1 }` | 50,000 → **8,899** | + +*(The test creates each index, measures, and drops it again, so your `orders` +collection is left exactly as it started.)* + +--- + +## What you get on managed Azure DocumentDB + +This local image demonstrates the **biggest** win — turning a full scan into an +index scan. Managed **Azure DocumentDB** goes further with two optimizations that +are enabled there but not in the local preview image: + +- **Index-backed sort** — the index returns rows already in order, removing the + in-memory sort step. +- **Covered queries** — when the index contains every field the query needs, the + database never touches the documents at all. + +So on Azure the same query gets *even* faster. Learn more: +[Query Performance Tuning Guide](https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/) · +[How to read explain output](https://learn.microsoft.com/azure/documentdb/how-to-read-explain-output). + +--- + +## Clean up + +```bash +docker rm -f documentdb-local +``` + +## Where to next + +- **Ask your assistant** to review the indexes on your own collections, or to + explain a slow query from your app — the same skills apply. +- **Find slow queries in production:** on Azure, route Diagnostic Logs to a Log + Analytics workspace and rank operations in the `VCoreMongoRequests` table by + `DurationMs`, then bring the worst offender back to your assistant. diff --git a/knowledge-base/README.md b/knowledge-base/README.md index 9213d3f..45cf3bc 100644 --- a/knowledge-base/README.md +++ b/knowledge-base/README.md @@ -1,7 +1,8 @@ # DocumentDB Agent-Kit — Knowledge Base Layer This is the layer that sits **above** the scripts and skills: it turns a -developer's **natural-language question** into the **exact diagnostic to run**. +developer's **natural-language question** into the **exact target to use** — a +read-only diagnostic **script** (Route A) or a text **skill** (Route B). ``` natural-language query @@ -11,13 +12,18 @@ developer's **natural-language question** into the **exact diagnostic to run**. │ knowledge base │ kb.json (single source of truth) │ + router │ kb-route.sh └────────┬─────────┘ - one hop │ multi hop (guarded workflow) - ┌────────────────┘ └───────────────┐ - ▼ ▼ - a single script step → (result?) → step → … → conclusion - (scripts/*.sh) (scripts/*.sh at each node) + Route A (scripts) │ Route B (skills) + multi-hop workflow (guarded) + ┌────────────────┴────────────────┐ + ▼ ▼ + a diagnostic script a skill's SKILL.md + (scripts/*.sh, measured facts) (skills/*/SKILL.md, guidance) ``` +The router scores a question against **both** spaces and reports the best of each +(`match` for scripts, `skill_match` for skills) plus a `recommended` route — so a +"run the TOAST advisor" question lands on a script, while a "how do I read explain +output" question lands on a skill. + Unlike a text-only skill kit (which hands the model prose and hopes it picks the right approach), this layer gives a **deterministic, explainable routing decision** and the ready-to-run command — while remaining fully consumable by an @@ -27,9 +33,9 @@ LLM agent for the semantic cases. | File | Role | |------|------| -| `kb.json` | Declarative KB: `tools` (scripts + intents), `routes_one_hop`, `workflow_schema`, and `workflows` (multi-hop, currently one scaffold). Edit this to extend the kit. | +| `kb.json` | Declarative KB: `tools` (Route A scripts + intents), `skills` (Route B text targets + intents), `routes_one_hop` / `routes_one_hop_skills`, `workflow_schema`, and `workflows` (multi-hop, currently one scaffold). Edit this to extend the kit. | | `kb-route.sh` | CLI wrapper (bash): arg parsing + presence checks; passes inputs to `kb_route.py` via env vars. | -| `kb_route.py` | Routing engine (stdlib python3, no deps): keyword/example scoring → best tool + exact command. Standalone so it can be linted/tested/imported. | +| `kb_route.py` | Routing engine (stdlib python3, no deps): keyword/example scoring → best script + exact command **and** best skill + `SKILL.md` to open. Standalone so it can be linted/tested/imported. | | `kb_route_demo.py` | Teaching/debug aid: prints the full scoring walkthrough (per-tool score + signal breakdown) and how the router lands on the winner. `python3 knowledge-base/kb_route_demo.py [query]`. | | `README.md` | This file. | @@ -44,20 +50,30 @@ bash knowledge-base/kb-route.sh --db mydb "audit my indexes for redundancy" bash knowledge-base/kb-route.sh --json --db mydb "is my cache hit ratio ok?" # discovery -bash knowledge-base/kb-route.sh --list # all tools + example queries +bash knowledge-base/kb-route.sh --list # all Route A scripts + example queries +bash knowledge-base/kb-route.sh --skills # all Route B skills + example queries bash knowledge-base/kb-route.sh --workflows # multi-hop workflows (schema/scaffold) ``` -Example: +Example (Route A — script): ``` Query: "is my cache hit ratio ok, do I need more shared_buffers" -→ Route: [db-config-advisor] Config & Cache Advisor (confidence: high, score 9.5) +→ Route A (script): [db-config-advisor] Config & Cache Advisor (confidence: high, score 9.5) ← recommended matched: cache, cache hit, shared_buffers run: bash scripts/db-config-advisor.sh --db mydb [--json] ``` -Currently routed tools (all present DocumentDB diagnostics): +Example (Route B — skill): + +``` +Query: "how do I read explain output and apply the ESR rule" +→ Route B (skill): [query-performance-tuning] Query Performance Tuning Guide (confidence: high, score 14.4) ← recommended + matched: explain, explain output, read explain, esr rule + open: skills/query-performance-tuning/SKILL.md +``` + +Currently routed tools (Route A — DocumentDB diagnostics): | Tool | Answers questions like | |------|------------------------| @@ -67,10 +83,23 @@ Currently routed tools (all present DocumentDB diagnostics): | `perf-advisor` | "performance checkup", "missing indexes / collection scans" | | `data-integrity-check` | "orphaned references", "referential integrity", "type consistency" | -**Routing is deterministic:** the router scores the query against each tool's +Currently routed skills (Route B — text guidance): + +| Skill | Answers questions like | +|------|------------------------| +| `query-performance-tuning` | "how do I read explain output", "what is the ESR rule", "find slow queries in prod" | +| `query-optimizer` | "optimize this query", "recommend an index", "verify this query uses an index / check the query plan" | +| `indexing` | "which index type should I use", "design a compound index", "multikey / wildcard / TTL index" | +| `natural-language-querying` | "write a query / aggregation", "translate this SQL to MongoDB" | +| `mcp-setup` | "set up the documentdb mcp server", "configure connection profiles" | +| `azure-deployment` | "provision a cluster with Bicep / Terraform", "get the connection string" | +| `connection` | "tune my connection pool", "maxPoolSize for serverless", "pool exhaustion" | + +**Routing is deterministic:** the router scores the query against each target's `keywords` / `example_queries` in `kb.json` (multi-word phrases weigh more than -single tokens) plus the `routes_one_hop` signal, and returns a ranked result -with a confidence and alternatives. No LLM required; same input → same route. +single tokens) plus the `routes_one_hop` / `routes_one_hop_skills` signal, and +returns a ranked result **per space** (best script, best skill) with a confidence, +alternatives, and a `recommended` route. No LLM required; same input → same route. ## Multi-hop workflows (schema + scaffold) @@ -99,17 +128,29 @@ slow-writes: ## Extending the KB -- **Add a one-hop tool:** append an entry to `tools[]` in `kb.json` with its - `script`, `invocation`, `keywords`, and `example_queries`. The router picks it - up automatically — no code change. +- **Add a one-hop tool (Route A script):** append an entry to `tools[]` in + `kb.json` with its `script`, `invocation`, `keywords`, and `example_queries`. + The router picks it up automatically — no code change. +- **Add a one-hop skill (Route B text target):** append an entry to `skills[]` + in `kb.json` with its `id`, `name`, `title`, `path` (`skills//SKILL.md`), + `keywords`, and `example_queries` — and, optionally, a couple of representative + queries to `routes_one_hop_skills.examples`. The router scores it automatically. - **Add a workflow:** append to `workflows[]` following `workflow_schema` (`entry`, `steps`, `depends_on`, guarded `yields`). +Routing is regression-guarded by the `kb-router` scenario +(`testing/scenarios/kb-router/`); add a row to its `expected-findings.yaml` +(`routes:` for a script, `skill_routes:` for a skill) when you add a target. + ## How the agent should use this layer -1. On a natural-language diagnostic question, call `kb-route.sh --json ""`. -2. If `confident`, run the emitted `command` (filling `--db`), then interpret the - script's measured output for the user. +1. On a natural-language question, call `kb-route.sh --json ""`. +2. Look at `recommended` (`"script"` or `"skill"`): + - **script** → if `confident`, run the emitted `match.command` (filling + `--db`), then interpret the script's measured output for the user. + - **skill** → if `skill_confident`, open the `skill_match.open` `SKILL.md` and + answer from its guidance (Route B — no script is run). 3. If a multi-hop workflow applies, start at its `entry` step and follow the guarded edges using each step's result. -4. If no confident route, fall back to `--list` and ask the user to clarify. +4. If neither space is confident, fall back to `--list` / `--skills` and ask the + user to clarify. diff --git a/knowledge-base/kb-route.sh b/knowledge-base/kb-route.sh index bc9334e..e73acba 100755 --- a/knowledge-base/kb-route.sh +++ b/knowledge-base/kb-route.sh @@ -1,11 +1,13 @@ #!/usr/bin/env bash # kb-route.sh — DocumentDB Agent-Kit knowledge-base router. # -# Maps a natural-language diagnostic question to the exact agent-kit script that -# answers it (ONE HOP: query -> script). Reads knowledge-base/kb.json as the -# single source of truth. Deterministic keyword/example scoring (stdlib only, no -# deps) so it works without an LLM — and gives the LLM agent a structured, -# reproducible routing decision it can trust and explain. +# Maps a natural-language question to the exact agent-kit target that answers it +# (ONE HOP). Two target spaces, both scored the same way: Route A `tools` +# (read-only scripts -> `bash scripts/*.sh`) and Route B `skills` (text guidance +# -> `skills/*/SKILL.md`). Reads knowledge-base/kb.json as the single source of +# truth. Deterministic keyword/example scoring (stdlib only, no deps) so it works +# without an LLM — and gives the LLM agent a structured, reproducible routing +# decision (best script + best skill + recommended route) it can trust. # # Multi-hop troubleshooting workflows (guarded diagnostic graph) are described by # the kb.json workflow_schema and listed with --workflows; traversal is left to @@ -14,8 +16,9 @@ # Usage: # bash knowledge-base/kb-route.sh "why are my writes slow?" # route # bash knowledge-base/kb-route.sh --db mydb "audit my indexes" # fill -# bash knowledge-base/kb-route.sh --json "check data integrity" # machine -# bash knowledge-base/kb-route.sh --list # all tools +# bash knowledge-base/kb-route.sh --json "how do I read explain" # machine +# bash knowledge-base/kb-route.sh --list # all scripts +# bash knowledge-base/kb-route.sh --skills # all skills # bash knowledge-base/kb-route.sh --workflows # workflows set -uo pipefail @@ -31,9 +34,10 @@ while [[ $# -gt 0 ]]; do --db) DB="$2"; shift 2;; --json) JSON=1; shift;; --list) MODE="list"; shift;; + --skills) MODE="skills"; shift;; --workflows) MODE="workflows"; shift;; --kb) KB="$2"; shift 2;; - -h|--help) sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; exit 0;; + -h|--help) sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//'; exit 0;; *) QUERY="${QUERY:+$QUERY }$1"; shift;; esac done diff --git a/knowledge-base/kb.json b/knowledge-base/kb.json index 52b520b..f759d47 100644 --- a/knowledge-base/kb.json +++ b/knowledge-base/kb.json @@ -1,11 +1,12 @@ { - "kb_version": "0.1.0", - "description": "DocumentDB Agent-Kit knowledge base. Single source of truth mapping natural-language diagnostic intents to the exact scripts that answer them (one hop), with a schema for multi-hop troubleshooting workflows (guarded diagnostic graph) to be populated later.", + "kb_version": "0.2.0", + "description": "DocumentDB Agent-Kit knowledge base. Single source of truth mapping natural-language intents to the exact target that answers them (one hop): a read-only diagnostic script (Route A, `tools`) OR a text skill (Route B, `skills`). Also holds a schema for multi-hop troubleshooting workflows (guarded diagnostic graph) to be populated later.", "conventions": { "db_placeholder": "", "run_prefix": "bash", "scripts_root": "scripts/", - "notes": "Every tool reads a live DocumentDB container locally (mongosh + psql). Recommendations are measured facts, never generic rules-of-thumb." + "skills_root": "skills/", + "notes": "Route A `tools` are read-only scripts that inspect a live DocumentDB container locally (mongosh + psql) and emit measured facts. Route B `skills` are prose guidance the agent opens/reads (no script run). The router scores a query against both spaces and reports the best of each (`match` for scripts, `skill_match` for skills)." }, "tools": [ @@ -107,6 +108,139 @@ ] }, + "skills": [ + { + "id": "query-performance-tuning", + "name": "documentdb-query-performance-tuning", + "title": "Query Performance Tuning Guide", + "path": "skills/query-performance-tuning/SKILL.md", + "route": "B", + "produces": "end-to-end tuning methodology: read DocumentDB explain output, ESR compound-index design, index-backed sorts, covered queries, finding slow queries via Log Analytics", + "keywords": ["explain", "explain output", "read explain", "executionstats", "execution stats", "esr rule", "equality sort range", "covered query", "covered queries", "collscan", "collection scan", "ixscan", "in-memory sort", "index-backed sort", "sort stage", "compound index order", "query performance tuning", "performance tuning guide", "tune query", "tune my query", "find slow queries", "slow query log", "vcoremongorequests", "diagnostic logs", "log analytics", "parallel_sort_merge", "indexcosts"], + "example_queries": [ + "how do I read the explain output in documentdb", + "what is the ESR rule for compound indexes", + "how do I tune a slow query end to end", + "how do I find slow queries in production with log analytics", + "what is a covered query and how do I get one", + "how do I stop my query from doing a collection scan" + ] + }, + { + "id": "query-optimizer", + "name": "documentdb-query-optimizer", + "title": "Query Optimizer (interactive)", + "path": "skills/query-optimizer/SKILL.md", + "route": "B", + "produces": "interactive, MCP-driven optimization of a specific query: verify the plan with explain (IXSCAN vs COLLSCAN), list indexes, propose and (with approval) create an ESR index; also validates index usage", + "keywords": ["optimize", "optimize query", "optimize this query", "index this", "index this query", "index review", "review my indexes", "recommend index", "index recommendation", "which index should I create", "add an index", "fix my slow query", "fix slow queries", "query optimization", "improve this query", "verify index usage", "check index usage", "is my query using an index", "does this query use an index", "does a collection scan", "avoid collscan", "avoid collection scan", "check the query plan", "query plan review", "verify the query uses an index", "explain plan"], + "example_queries": [ + "optimize this query and recommend an index", + "how should I index this specific query", + "review the indexes on my collection and suggest improvements", + "recommend an index for this find and sort", + "use explain to check whether this query uses an index or does a collection scan", + "verify that my query is using an index and not scanning the collection" + ] + }, + { + "id": "natural-language-querying", + "name": "documentdb-natural-language-querying", + "title": "Natural-Language Querying (SQL → MQL)", + "path": "skills/natural-language-querying/SKILL.md", + "route": "B", + "produces": "read-only find/aggregation queries generated from natural language, including SQL-to-MongoDB translation", + "keywords": ["write a query", "generate a query", "how do I query", "query syntax", "aggregation pipeline", "aggregate", "aggregation", "filter documents", "group by", "sql to mongodb", "translate sql", "find documents", "mql", "write an aggregation", "query for documents"], + "example_queries": [ + "how do I write a query to filter documents by status", + "generate an aggregation pipeline to group orders by month", + "translate this SQL query into a MongoDB query", + "how do I query for documents where the amount is greater than 100" + ] + }, + { + "id": "mcp-setup", + "name": "documentdb-mcp-setup", + "title": "MCP Server Setup", + "path": "skills/mcp-setup/SKILL.md", + "route": "B", + "produces": "install/configure the DocumentDB MCP server in an agentic client and define CONNECTION_PROFILES; fix MCP connection/auth/profile errors", + "keywords": ["mcp", "mcp server", "documentdb mcp", "connection profiles", "connection_profiles", "configure mcp", "install mcp", "set up mcp", "setup mcp", "mcp auth", "mcp connection error", "claude code", "cursor mcp", "copilot cli", "gemini cli", "mcp profile", "wire up mcp"], + "example_queries": [ + "how do I set up the documentdb mcp server", + "configure the mcp server connection profiles", + "install the documentdb mcp server in cursor", + "my mcp server authentication is failing" + ] + }, + { + "id": "azure-deployment", + "name": "documentdb-azure-deployment", + "title": "Azure Cluster Deployment", + "path": "skills/azure-deployment/SKILL.md", + "route": "B", + "produces": "provision an Azure DocumentDB cluster (Microsoft.DocumentDB/mongoClusters) via Bicep / Azure CLI / Terraform / portal, plus firewall rules, connection string retrieval, teardown", + "keywords": ["deploy", "deploy cluster", "provision", "provision cluster", "create cluster", "spin up", "spin up cluster", "bicep", "terraform", "azure cli", "mongoclusters", "mongo cluster", "infrastructure as code", "firewall rule", "cluster tier", "teardown", "connection string"], + "example_queries": [ + "how do I provision an azure documentdb cluster", + "deploy a documentdb cluster with bicep", + "spin up a new cluster with terraform", + "how do I create a mongo cluster and get its connection string" + ] + }, + { + "id": "connection", + "name": "documentdb-connection", + "title": "Connection / Pool Tuning", + "path": "skills/connection/SKILL.md", + "route": "B", + "produces": "MongoClient connection-pool / timeout / retry tuning for serverless, OLTP, OLAP, or bursty workloads; connection-error troubleshooting", + "keywords": ["connection pool", "pool size", "maxpoolsize", "connection timeout", "connection tuning", "econnrefused", "pool exhaustion", "mongoclient", "singleton client", "retry writes", "serverless connection", "connection error", "timeout tuning", "connection settings", "connection pooling"], + "example_queries": [ + "how do I tune my connection pool size", + "what maxPoolSize should I use for a serverless function", + "I'm getting connection timeouts and pool exhaustion errors", + "how should I configure the mongo client connection for high traffic" + ] + }, + { + "id": "indexing", + "name": "documentdb-indexing", + "title": "Indexing — index-type selection & design", + "path": "skills/indexing/SKILL.md", + "route": "B", + "produces": "choosing the right index TYPE (single / compound / multikey / wildcard / hashed / 2dsphere / TTL), ESR compound-index design, query-pattern -> index-shape cookbook, and the safe hideIndex -> dropIndex lifecycle", + "keywords": ["index type", "which index type", "design an index", "design this index", "design a compound index", "compound index", "single field index", "multikey index", "wildcard index", "hashed index", "2dsphere index", "geospatial index", "ttl index", "index shape", "index cookbook", "hide index", "hideindex", "index lifecycle", "esr ordering", "what index do I need"], + "example_queries": [ + "which index type should I use for this query", + "design a compound index for this filter and sort", + "do I need a multikey or wildcard index", + "which compound index should I design for this filter, sort, and range query", + "how do I safely hide then drop an index" + ] + } + ], + + "routes_one_hop_skills": { + "description": "Representative NL query -> skill id (Route B). Extra routing signal for the skill space, mirroring routes_one_hop for tools.", + "examples": [ + { "query": "how do I read explain output and apply the ESR rule", "skill": "query-performance-tuning" }, + { "query": "how do I find slow queries in production", "skill": "query-performance-tuning" }, + { "query": "optimize this query and recommend an index", "skill": "query-optimizer" }, + { "query": "how should I index this specific query", "skill": "query-optimizer" }, + { "query": "which index type should I use for this query", "skill": "indexing" }, + { "query": "design a compound index for this filter sort and range", "skill": "indexing" }, + { "query": "verify this query uses an index and not a collection scan", "skill": "query-optimizer" }, + { "query": "check the query plan for a collection scan", "skill": "query-optimizer" }, + { "query": "help me write an aggregation pipeline", "skill": "natural-language-querying" }, + { "query": "translate this sql to mongodb", "skill": "natural-language-querying" }, + { "query": "set up the documentdb mcp server", "skill": "mcp-setup" }, + { "query": "provision an azure documentdb cluster with bicep", "skill": "azure-deployment" }, + { "query": "tune my connection pool size", "skill": "connection" }, + { "query": "fix connection pool exhaustion", "skill": "connection" } + ] + }, + "workflow_schema": { "description": "Schema for MULTI-HOP troubleshooting workflows: a guarded diagnostic graph (AND/OR). Each step runs a tool; edges are conditional on the observed result. The LLM agent presents a step, the human runs it (or the agent runs the script), and the reported result selects the next step or a conclusion. This layer is scaffolded; workflows are added over time.", "fields": { diff --git a/knowledge-base/kb_route.py b/knowledge-base/kb_route.py index 32c7949..a8eb821 100644 --- a/knowledge-base/kb_route.py +++ b/knowledge-base/kb_route.py @@ -1,11 +1,17 @@ #!/usr/bin/env python3 """kb_route.py — DocumentDB Agent-Kit knowledge-base router (routing engine). -Maps a natural-language diagnostic question to the exact agent-kit script that -answers it (ONE HOP: query -> script). Reads knowledge-base/kb.json as the single -source of truth. Deterministic keyword/example scoring (stdlib only, no deps) so -it works without an LLM — and gives the LLM agent a structured, reproducible -routing decision it can trust and explain. +Maps a natural-language question to the exact agent-kit target that answers it +(ONE HOP). There are two target spaces, both scored the same deterministic way +(keyword/example scoring, stdlib only, no deps): + + * Route A — `tools` : read-only diagnostic scripts (query -> `bash scripts/*.sh`) + * Route B — `skills` : text guidance the agent opens (query -> `skills/*/SKILL.md`) + +kb.json is the single source of truth. The router reports the best of each space +(`match` for scripts, `skill_match` for skills) and a `recommended` route, so it +works without an LLM and gives the LLM agent a structured, reproducible decision +it can trust and explain. This file is invoked by kb-route.sh (the CLI wrapper), which passes inputs via environment variables: KB, DB, JSON, MODE, QUERY. It is a standalone module so it @@ -70,8 +76,9 @@ def score_tool(t, q_tokens, q_lower): def rank_tools(kb, query): - """Rank all tools for a query. Returns (ranked, q_tokens, q_lower) where - ranked is a list of (score, tool, matched_keywords) sorted best-first.""" + """Rank all tools (Route A scripts) for a query. Returns (ranked, q_tokens, + q_lower) where ranked is a list of (score, tool, matched_keywords) sorted + best-first.""" q_tokens = tokenize(query) - STOP q_lower = query.lower() @@ -93,6 +100,36 @@ def rank_tools(kb, query): return ranked +def rank_skills(kb, query): + """Rank all skills (Route B text targets) for a query, mirroring rank_tools. + + Uses the same additive scoring (score_tool) over each skill's + keywords/example_queries, plus a routes_one_hop_skills boost. Returns a list + of (score, skill, matched_keywords) sorted best-first, or [] if the KB + defines no skills.""" + skills = kb.get("skills", []) + if not skills: + return [] + q_tokens = tokenize(query) - STOP + q_lower = query.lower() + + route_boost = {} + for r in kb.get("routes_one_hop_skills", {}).get("examples", []): + r_tokens = tokenize(r["query"]) - STOP + ov = len(r_tokens & q_tokens) + frac = ov / max(1, len(r_tokens)) + if frac > route_boost.get(r["skill"], 0): + route_boost[r["skill"]] = frac + + ranked = [] + for sk in skills: + s, hits = score_tool(sk, q_tokens, q_lower) + s += 2.0 * route_boost.get(sk["id"], 0.0) + ranked.append((s, sk, hits)) + ranked.sort(key=lambda x: -x[0]) + return ranked + + def run_list(kb, db, placeholder, as_json): if as_json: print(json.dumps( @@ -112,6 +149,28 @@ def run_list(kb, db, placeholder, as_json): return 0 +def run_skills_list(kb, as_json): + skills = kb.get("skills", []) + if as_json: + print(json.dumps( + [{"id": sk["id"], "name": sk.get("name"), "title": sk["title"], + "open": sk["path"], "produces": sk.get("produces")} + for sk in skills], indent=2)) + return 0 + if not skills: + print("No skills registered for routing yet. See kb.json skills[] to add one.") + return 0 + print("Knowledge base skills (Route B — one-hop text targets):") + for sk in skills: + print(f"\n [{sk['id']}] {sk['title']} ({sk.get('name','')})") + print(f" open: {sk['path']}") + print(f" for: {sk.get('produces','')}") + ex = sk.get("example_queries", []) + if ex: + print(f" e.g. \"{ex[0]}\"") + return 0 + + def run_workflows(kb, as_json): wfs = kb.get("workflows", []) if as_json: @@ -138,7 +197,7 @@ def run_workflows(kb, as_json): def run_route(kb, db, placeholder, as_json, query): if not query: - print("Provide a natural-language query, or use --list / --workflows.", + print("Provide a natural-language query, or use --list / --skills / --workflows.", file=sys.stderr) return 2 @@ -146,6 +205,19 @@ def run_route(kb, db, placeholder, as_json, query): best_s, best_t, best_hits = ranked[0] alternatives = [(s, t) for s, t, _ in ranked[1:] if s > 0][:2] + ranked_sk = rank_skills(kb, query) + best_sk_s, best_sk, best_sk_hits = ranked_sk[0] if ranked_sk else (0.0, None, []) + sk_alternatives = [(s, sk) for s, sk, _ in ranked_sk[1:] if s > 0][:2] + + # Which route (script vs skill) is the stronger answer? Ties favour the + # script (Route A) since a measured diagnostic beats generic guidance. + if best_sk_s > best_s and best_sk_s > 0: + recommended = "skill" + elif best_s > 0: + recommended = "script" + else: + recommended = None + if as_json: out = { "query": query, @@ -157,27 +229,47 @@ def run_route(kb, db, placeholder, as_json, query): }, "alternatives": [{"tool": t["id"], "score": round(s, 2)} for s, t in alternatives], "confident": best_s >= 2.0, + "skill_match": None if best_sk_s <= 0 else { + "skill": best_sk["id"], "name": best_sk.get("name"), "title": best_sk["title"], + "score": round(best_sk_s, 2), "matched_keywords": best_sk_hits, + "open": best_sk["path"], + }, + "skill_alternatives": [{"skill": sk["id"], "score": round(s, 2)} for s, sk in sk_alternatives], + "skill_confident": best_sk_s >= 2.0, + "recommended": recommended, } print(json.dumps(out, indent=2)) return 0 - if best_s <= 0: + if best_s <= 0 and best_sk_s <= 0: print(f'No confident route for: "{query}"') - print("Available tools (use --list for details):") - for t in kb["tools"]: - print(f" - {t['id']}: {t.get('produces','')}") + print("Available scripts (use --list) / skills (use --skills).") return 0 - conf = "high" if best_s >= 4 else ("medium" if best_s >= 2 else "low") print(f'Query: "{query}"') - print(f'→ Route: [{best_t["id"]}] {best_t["title"]} (confidence: {conf}, score {best_s:.1f})') - if best_hits: - print(f' matched: {", ".join(best_hits[:6])}') - print(f' run: {fill(best_t["invocation"], db, placeholder)}') - if (not db) and (placeholder in best_t["invocation"]): - print(f' (supply the database: --db — replaces {placeholder})') - if alternatives: - print(" alternatives: " + ", ".join(f"{t['id']} ({s:.1f})" for s, t in alternatives)) + + if best_s > 0: + conf = "high" if best_s >= 4 else ("medium" if best_s >= 2 else "low") + tag = " ← recommended" if recommended == "script" else "" + print(f'→ Route A (script): [{best_t["id"]}] {best_t["title"]} (confidence: {conf}, score {best_s:.1f}){tag}') + if best_hits: + print(f' matched: {", ".join(best_hits[:6])}') + print(f' run: {fill(best_t["invocation"], db, placeholder)}') + if (not db) and (placeholder in best_t["invocation"]): + print(f' (supply the database: --db — replaces {placeholder})') + if alternatives: + print(" alternatives: " + ", ".join(f"{t['id']} ({s:.1f})" for s, t in alternatives)) + + if best_sk_s > 0: + conf = "high" if best_sk_s >= 4 else ("medium" if best_sk_s >= 2 else "low") + tag = " ← recommended" if recommended == "skill" else "" + print(f'→ Route B (skill): [{best_sk["id"]}] {best_sk["title"]} (confidence: {conf}, score {best_sk_s:.1f}){tag}') + if best_sk_hits: + print(f' matched: {", ".join(best_sk_hits[:6])}') + print(f' open: {best_sk["path"]}') + if sk_alternatives: + print(" alternatives: " + ", ".join(f"{sk['id']} ({s:.1f})" for s, sk in sk_alternatives)) + return 0 @@ -195,6 +287,8 @@ def main(): if mode == "list": return run_list(kb, db, placeholder, as_json) + if mode == "skills": + return run_skills_list(kb, as_json) if mode == "workflows": return run_workflows(kb, as_json) return run_route(kb, db, placeholder, as_json, query) diff --git a/scenarios/ecommerce/README.md b/scenarios/ecommerce/README.md index 87913d8..8deec6b 100644 --- a/scenarios/ecommerce/README.md +++ b/scenarios/ecommerce/README.md @@ -20,8 +20,15 @@ bash scenarios/ecommerce/seed.sh # -> database "ecommerce" bash scripts/perf-advisor.sh --db ecommerce bash scripts/index-redundancy-finder.sh --db ecommerce bash scripts/data-integrity-check.sh --db ecommerce + +# query-performance skill before/after test (COLLSCAN -> IXSCAN on orders) +bash scenarios/ecommerce/query-perf-skill-test.sh ``` +The query-perf test harness (`query-perf-skill-test.js` / `.sh`) validates the +query-performance skills against this dataset; see the generated instruction doc +under [`docs/`](../../docs/) (`query-perf-skill-test-instruction-*.md`). + Prereq: a running `documentdb-local` container — see the repo [`README.md`](../../README.md#quickstart) *Quickstart*. Overrides: `--container`, `--password`, `--db`, or the `DB_USER`/`DB_PASSWORD`/`PORT` env vars. diff --git a/scenarios/ecommerce/query-perf-skill-test.js b/scenarios/ecommerce/query-perf-skill-test.js new file mode 100644 index 0000000..f1802e0 --- /dev/null +++ b/scenarios/ecommerce/query-perf-skill-test.js @@ -0,0 +1,157 @@ +// query-perf-skill-test.js — before/after query-performance harness for the +// "Query Performance Tuning Guide" skills, run against the seeded `ecommerce` +// dataset (orders: 50,000 docs, no secondary index by default). +// +// For each skill it: (1) drops every non-_id index on `orders` to force the +// COLLSCAN baseline, (2) measures the slow query with explain("executionStats"), +// (3) creates the skill's recommended index, (4) re-measures, then drops it. +// +// It reports the honest "work" metric the article tells you to read — the +// documents/keys examined AT THE SCAN STAGE (not the misleading top-line count) — +// plus the winning plan shape and a min-of-5 executionTimeMillis. +// +// Run via: bash scenarios/ecommerce/query-perf-skill-test.sh +// (or: mongosh /ecommerce --file query-perf-skill-test.js) + +const COLL = "orders"; +const RUNS = 5; + +// ── helpers ───────────────────────────────────────────────────────────────── +function N(x) { // DocumentDB returns Long as {low,high} + if (x && typeof x === "object" && "low" in x) return x.low; + return typeof x === "number" ? x : 0; +} +function stagesOf(root) { // flatten the inputStage chain + const out = []; + for (let s = root; s; s = s.inputStage) out.push(s); + return out; +} +function measure(filter, sort, projection) { + let best = null, tMin = Infinity; + for (let i = 0; i < RUNS; i++) { + let cur = db[COLL].find(filter, projection || {}); + if (sort) cur = cur.sort(sort); + const e = cur.explain("executionStats"); + const es = e.executionStats; + const chain = stagesOf(es.executionStages); + const scan = chain[chain.length - 1]; // deepest stage + const t = N(es.executionTimeMillis); + if (t < tMin) tMin = t; + if (!best) { + const fetch = chain.find(s => s.stage === "FETCH"); + best = { + planTop: es.executionStages.stage, + scanStage: scan.stage, + scanIndex: scan.indexName || null, + // COLLSCAN reports work as totalDocsExamined; IXSCAN as totalKeysExamined + scanExamined: scan.stage === "COLLSCAN" ? N(scan.totalDocsExamined) + : N(scan.totalKeysExamined), + scanExaminedField: scan.stage === "COLLSCAN" ? "docsExamined" : "keysExamined", + docsFetched: fetch ? N(fetch.totalDocsExamined) : 0, + hasSort: chain.some(s => s.stage === "SORT"), + hasFetch: !!fetch, + topDocsExamined: N(es.totalDocsExamined), // the misleading top-line + nReturned: N(es.nReturned), + }; + } + } + best.timeMsMin = tMin; + return best; +} +function dropSecondary() { + db[COLL].getIndexes().filter(i => i.name !== "_id_") + .forEach(i => db[COLL].dropIndex(i.name)); +} +function fmt(m) { + const sort = m.hasSort ? " +SORT" : ""; + const fetch = m.hasFetch ? " +FETCH" : ""; + return `${m.scanStage}${m.scanIndex ? "(" + m.scanIndex + ")" : ""}` + + ` ${m.scanExaminedField}=${m.scanExamined}${sort}${fetch}` + + ` nReturned=${m.nReturned} time=${m.timeMsMin}ms (top docsExamined=${m.topDocsExamined})`; +} + +// ── the four skill test cases ──────────────────────────────────────────────── +const TESTS = [ + { + skill: "documentdb-query-optimizer", + rule: "explain-plan verification (merged from the former query-optimization skill)", + point: "explain() literacy: a COLLSCAN becomes an IXSCAN once an index supports the filter", + trigger: "use explain to check whether this query uses an index or does a collection scan", + filter: { order_id: "ORD_012345" }, + sort: null, + index: { order_id: 1 }, + }, + { + skill: "documentdb-query-optimizer", + rule: "(standalone skill)", + point: "recommend a compound index for a specific slow query (two equality fields + sort)", + trigger: "optimize this query and recommend an index", + filter: { status: "shipped", shipping_city: "Seattle" }, + sort: { created_at: -1 }, + index: { status: 1, shipping_city: 1, created_at: -1 }, + }, + { + skill: "documentdb-query-performance-tuning", + rule: "(standalone skill) — the article's flagship query", + point: "full ESR compound index, most-selective equality first; + covered-query variant", + trigger: "how do I read the explain output and apply the ESR rule to tune this query", + filter: { status: "shipped", customer_id: "CUST_004087", created_at: { $gte: ISODate("2024-01-01") } }, + sort: { created_at: -1 }, + index: { customer_id: 1, status: 1, created_at: -1 }, + covered: { _id: 0, customer_id: 1, status: 1, created_at: 1 }, + }, + { + skill: "documentdb-indexing", + rule: "index-compound-esr", + point: "ESR with a range predicate: Equality, then Sort, then Range LAST", + trigger: "which compound index should I design for this filter, sort, and range query", + filter: { status: "shipped", total_amount: { $gt: 500 } }, + sort: { created_at: -1 }, + index: { status: 1, created_at: -1, total_amount: 1 }, + }, +]; + +// ── run ────────────────────────────────────────────────────────────────────── +print("=".repeat(78)); +print(" Query-Performance Skill Test — dataset: " + db.getName() + "." + COLL + + " (" + db[COLL].countDocuments() + " docs)"); +print("=".repeat(78)); + +const summary = []; +for (const t of TESTS) { + dropSecondary(); + const before = measure(t.filter, t.sort); + const idxName = db[COLL].createIndex(t.index); + sleep(500); + const after = measure(t.filter, t.sort); + let covered = null; + if (t.covered) covered = measure(t.filter, t.sort, t.covered); + dropSecondary(); + + print("\n### " + t.skill + " [" + t.rule + "]"); + print(" goal : " + t.point); + print(" trigger : \"" + t.trigger + "\""); + print(" query : db.orders.find(" + JSON.stringify(t.filter) + + (t.sort ? ").sort(" + JSON.stringify(t.sort) : "") + ")"); + print(" index : db.orders.createIndex(" + JSON.stringify(t.index) + ") [" + idxName + "]"); + print(" BEFORE : " + fmt(before)); + print(" AFTER : " + fmt(after)); + if (covered) print(" COVERED : " + fmt(covered)); + const reduction = after.scanExamined > 0 + ? (before.scanExamined / after.scanExamined).toFixed(0) + "x fewer examined at the scan" + : "n/a"; + print(" RESULT : scan work " + before.scanExamined + " -> " + after.scanExamined + + " (" + reduction + ")"); + + summary.push({ + skill: t.skill, rule: t.rule, trigger: t.trigger, + filter: t.filter, sort: t.sort, index: t.index, indexName: idxName, + before, after, covered, + scanWorkReduction: after.scanExamined > 0 ? before.scanExamined / after.scanExamined : null, + }); +} + +print("\n" + "---JSON-START---"); +print(JSON.stringify({ dataset: db.getName(), collection: COLL, + docCount: db[COLL].countDocuments(), results: summary }, null, 1)); +print("---JSON-END---"); diff --git a/scenarios/ecommerce/query-perf-skill-test.sh b/scenarios/ecommerce/query-perf-skill-test.sh new file mode 100755 index 0000000..0969107 --- /dev/null +++ b/scenarios/ecommerce/query-perf-skill-test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# query-perf-skill-test.sh — run the query-performance skill before/after harness +# against the seeded `ecommerce` dataset in a local DocumentDB container. +# +# Prereq: a running `documentdb-local` container with the ecommerce dataset seeded +# export DB_PASSWORD=Test1234 +# bash scenarios/ecommerce/seed.sh +# bash scenarios/ecommerce/query-perf-skill-test.sh +# +# Read-only against your data except that it CREATES then DROPS its own test +# indexes on `orders` (leaving the collection with only its default _id index). +set -uo pipefail + +CONTAINER_NAME="${CONTAINER_NAME:-documentdb-local}" +PORT="${PORT:-10260}" +USER="${DB_USER:-docdbadmin}" +PASSWORD="${DB_PASSWORD:-}" +DB="ecommerce" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +JS="$HERE/query-perf-skill-test.js" + +while [[ $# -gt 0 ]]; do + case "$1" in + --container) CONTAINER_NAME="$2"; shift 2;; + --password) PASSWORD="$2"; shift 2;; + --port) PORT="$2"; shift 2;; + --db) DB="$2"; shift 2;; + -h|--help) echo "Usage: $0 [--container NAME] [--password PASS] [--port PORT] [--db NAME]"; exit 0;; + *) shift;; + esac +done + +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -f "$JS" ]] || { echo "Error: harness not found at $JS" >&2; exit 1; } + +# Copy the harness into the container and run it with the bundled mongosh. +docker cp "$JS" "${CONTAINER_NAME}:/tmp/query-perf-skill-test.js" >/dev/null +docker exec -u documentdb "$CONTAINER_NAME" mongosh \ + "localhost:${PORT}/${DB}" -u "$USER" -p "$PASSWORD" \ + --authenticationMechanism SCRAM-SHA-256 --tls --tlsAllowInvalidCertificates \ + --quiet --file /tmp/query-perf-skill-test.js diff --git a/skills/data-modeling/model-large-field-split.md b/skills/data-modeling/model-large-field-split.md index 239b542..1465f4e 100644 --- a/skills/data-modeling/model-large-field-split.md +++ b/skills/data-modeling/model-large-field-split.md @@ -71,4 +71,4 @@ threshold after the split, and prints safe (batched, copy-before-delete, then ## References - Related: [model-embed-vs-reference](model-embed-vs-reference.md) (reference unbounded / independently-accessed data), [model-16mb-limit](model-16mb-limit.md). -- `storage/` skill for the PostgreSQL storage layer; `query-optimization/` for verifying scan cost with `explain("executionStats")`. +- `storage/` skill for the PostgreSQL storage layer; `query-optimizer/` for verifying scan cost with `explain("executionStats")`. diff --git a/skills/monitoring/monitoring-slow-query-log.md b/skills/monitoring/monitoring-slow-query-log.md index 059f60e..5582fc5 100644 --- a/skills/monitoring/monitoring-slow-query-log.md +++ b/skills/monitoring/monitoring-slow-query-log.md @@ -18,18 +18,30 @@ Relying on ad-hoc `db.currentOp()` checks during incidents with no historical lo ## Correct -Kusto example (exact table/column names depend on the current diagnostic schema — verify in your workspace): +Kusto example against the `VCoreMongoRequests` diagnostic log table — rank the slowest operations, then run `explain()` on each candidate: ```kusto -// Top 20 slow operations in the last 24h -DocumentDBSlowQueries_CL // or the current table name +// Top 20 slowest operations (> 1s) — copy PiiCommandText and run explain() on it +VCoreMongoRequests +| where DurationMs > 1000 +| project TimeGenerated, DatabaseName, CollectionName, + OperationName, DurationMs, PiiCommandText +| order by DurationMs desc +| take 20 +``` + +Key columns: `DurationMs` (execution time), `OperationName` (`find` / `aggregate` / `update` / …), `CollectionName`, and `PiiCommandText` (the actual command that ran). Copy `PiiCommandText` for a candidate and run `explain("executionStats")` on it — see the `documentdb-query-performance-tuning` skill (and its [`references/documentdb-explain-output.md`](../query-performance-tuning/references/documentdb-explain-output.md)) for how to read Azure DocumentDB's explain output. + +To aggregate by query shape over a window instead: + +```kusto +VCoreMongoRequests | where TimeGenerated > ago(24h) | summarize count(), avg_duration_ms = avg(DurationMs), - total_duration_ms = sum(DurationMs), - avg_docs_examined = avg(DocsExamined) - by Namespace, QueryShape + total_duration_ms = sum(DurationMs) + by DatabaseName, CollectionName, OperationName | order by total_duration_ms desc | take 20 ``` @@ -38,5 +50,6 @@ Action items from this dashboard typically include: adding a compound index, adj ## References +- [Query Performance Tuning Guide](https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/) (devblog) +- [How to monitor diagnostics logs](https://learn.microsoft.com/en-us/azure/documentdb/how-to-monitor-diagnostics-logs) - [Azure Monitor for Azure DocumentDB](https://learn.microsoft.com/azure/documentdb/) -- [MQL compatibility](https://learn.microsoft.com/azure/documentdb/compatibility-query-language) diff --git a/skills/query-optimization/SKILL.md b/skills/query-optimization/SKILL.md deleted file mode 100644 index d505446..0000000 --- a/skills/query-optimization/SKILL.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: documentdb-query-optimization -description: Query and aggregation-pipeline optimization rules for Azure DocumentDB — using `explain("executionStats")` to verify index usage and avoid `COLLSCAN`. Use when reviewing a specific query, diagnosing a slow query, or validating that an index is actually being used. For full index-design workflow, see the `documentdb-query-optimizer` skill. -license: MIT ---- - -# Query & Aggregation Optimization — Azure DocumentDB - -Best-practice rules for writing queries that can actually use indexes. For the full diagnostic workflow (explain output interpretation, ESR compound index design, covered queries, anti-patterns), see the `documentdb-query-optimizer` skill. - -## Rules - -- [query-explain-plan](query-explain-plan.md) — Use `explain("executionStats")` to verify index usage; watch `keysExamined` / `docsExamined` vs `nReturned`; avoid `COLLSCAN`. diff --git a/skills/query-optimization/query-explain-plan.md b/skills/query-optimization/query-explain-plan.md deleted file mode 100644 index d7fd3f2..0000000 --- a/skills/query-optimization/query-explain-plan.md +++ /dev/null @@ -1,30 +0,0 @@ -# query-explain-plan - -**Category:** Query & Aggregation Optimization · **Priority:** HIGH - -## Why it matters - -Guessing query performance is unreliable. `explain("executionStats")` reveals whether a query used an index (`IXSCAN`), did a collection scan (`COLLSCAN`), was targeted to a shard, or was scatter-gather. Use it as part of PR review for any new hot-path query. - -## Incorrect - -```javascript -// Ship it, hope it's fast in prod. -const results = await db.orders.find(filter).sort(sort).toArray(); -``` - -## Correct - -```javascript -const plan = await db.orders.find(filter).sort(sort).explain("executionStats"); -// Review: -// - winningPlan.stage: IXSCAN (good) vs COLLSCAN (bad) -// - executionStats.totalDocsExamined vs nReturned (ratio close to 1 is ideal) -// - shards[]: all shards hit => scatter-gather, fix with shard key in filter -``` - -Automate in CI with a small harness that asserts `totalDocsExamined / nReturned < threshold` for critical queries. - -## References - -- MongoDB [`explain()`](https://www.mongodb.com/docs/manual/reference/method/cursor.explain/) docs diff --git a/skills/query-optimizer/SKILL.md b/skills/query-optimizer/SKILL.md index 10000f6..b7ad0cd 100644 --- a/skills/query-optimizer/SKILL.md +++ b/skills/query-optimizer/SKILL.md @@ -2,11 +2,13 @@ name: documentdb-query-optimizer description: >- Help with DocumentDB/MongoDB query optimization and indexing for Azure - DocumentDB. Use only when the user asks for optimization or - performance: "How do I optimize this query?", "How do I index this?", "Why is - this query slow?", "Can you fix my slow queries?", etc. Do not invoke for - general query writing unless user asks for performance or index help. Prefer - indexing as optimization strategy. Use DocumentDB MCP when available. + DocumentDB. Use when the user asks for optimization or performance: "How do I + optimize this query?", "How do I index this?", "Why is this query slow?", "Can + you fix my slow queries?" — and also to **verify/validate** a query with + `explain("executionStats")`: "is this query using an index or doing a + COLLSCAN?", "check the query plan", "verify index usage". Do not invoke for + general query writing unless the user asks for performance or index help. + Prefer indexing as the optimization strategy. Use DocumentDB MCP when available. allowed-tools: mcp__documentdb__* --- @@ -20,6 +22,8 @@ Invoke **only** when the user wants: - **Why** a query is slow or **how to speed it up** - **Slow queries** on their cluster and/or **how to optimize them** - Index recommendations or index review +- To **verify** a query's plan: is it an `IXSCAN` or a `COLLSCAN`? is an index + actually being used? (validating index usage, e.g. in code review / CI) Do **not** invoke for routine query authoring unless the user has requested help with optimization, slow queries, or indexing. @@ -28,17 +32,32 @@ with optimization, slow queries, or indexing. ### Help with a Specific Query -If the user is asking about a particular query: - -1. Use `list_indexes` (MCP) or `db..getIndexes()` (mongosh) to get existing indexes on the collection -2. Use `explain_operation` (MCP) or `.explain("executionStats")` (mongosh) - to get explain output with execution stats -3. Use `find_documents` (MCP) or `db..findOne()` (mongosh) to fetch a sample document to understand the - schema - -Then make an optimization suggestion based on collected information and best -practices from the reference files. Prefer creating an index that fully covers -the query if possible. +**Order matters — check the plan first, and don't be fooled by confounders.** +The very first thing to establish is *what the query is actually doing*, because +the signals you read first are the ones most likely to mislead you. + +1. **Run `explain("executionStats")` first and read the SCAN stage.** Use + `explain_operation` (MCP) or `.explain("executionStats")` (mongosh). Determine + whether the query is a `COLLSCAN` or an `IXSCAN` **before** changing anything. + Watch these **confounders** — do not trust them at face value: + - ⚠️ **The top-line metrics lie.** On Azure DocumentDB the top-level + `executionTimeMillis` and `totalDocsExamined` are *post-sort/merge* and can + look fine during a full scan. **Drill into the nested `inputStage`** to the + `COLLSCAN`/`IXSCAN` and read *its* `totalDocsExamined`/`totalKeysExamined`. + - ⚠️ **An index existing ≠ the index being used.** Check `winningPlan.indexName`; + the planner may ignore an index that isn't selective for this shape. + - ⚠️ **Scatter-gather looks like a slow query.** On a sharded collection, + `shards[]` hitting every shard is the real cost — fix with the shard key in + the filter, not another index. +2. **Then list existing indexes** — `list_indexes` (MCP) or `db..getIndexes()` + (mongosh) — to know what's already available before recommending a new one. +3. **Then sample a document** — `find_documents` (MCP) or `db..findOne()` + (mongosh) — to understand the schema and field **cardinality** (selectivity), + which drives compound-index field order. + +Only after 1–3 do you diagnose and make an optimization suggestion, using the +best practices in the reference files. Prefer an index that fully covers the +query if possible. ### General Performance Help @@ -51,6 +70,12 @@ suggestions (not regarding any particular query): 4. Use `current_ops` (MCP) or `db.currentOp()` (mongosh) to see currently running operations 5. Suggest reviewing the most-used collections for missing indexes +To surface the slowest queries in **production**, use Diagnostic Logs routed to +an Azure Log Analytics workspace and rank operations from the +`VCoreMongoRequests` table by `DurationMs` — see the +`documentdb-query-performance-tuning` skill (Step 0) and the +`documentdb-monitoring` skill. + ## MCP Tools Available When DocumentDB MCP server is connected, these tools are available: @@ -104,12 +129,22 @@ Before beginning diagnosis and recommendation, load reference files. Always load: - `references/core-indexing-principles.md` +- `references/query-explain-plan.md` — verify index usage with `explain()` first, + and the confounders to distrust (top-line metrics, unused existing indexes, + scatter-gather); how to automate the check in CI. + +For the end-to-end tuning **methodology** (find slow queries via Log Analytics +`VCoreMongoRequests`, read DocumentDB's Postgres-backed `explain` output, the +`COLLSCAN → ESR → covered` walkthrough), see the companion +**`documentdb-query-performance-tuning`** skill and its +`references/documentdb-explain-output.md`. ## Diagnostic Workflow ### Step 1: Gather Information -For a specific query, run these tools: +For a specific query, run these tools **in this order — `explain` first** (see +"High Level Workflow" for the confounders to watch while reading it): **Via MCP (when connected):** ``` @@ -156,6 +191,15 @@ db..aggregate().explain("executionStats") From the `explain("executionStats")` response (via MCP `explain_operation` or direct mongosh), extract: +> **Reading a real Azure DocumentDB plan:** its planner is PostgreSQL-backed, so +> the output (`explainVersion: 2`) carries fields community MongoDB does not — +> `PARALLEL_SORT_MERGE`, `startupCost`/`totalCost`, `estimatedTotalKeysExamined`, +> `runtimeFilterSet`, `numBlocksFromCache`, `indexUsage.scanLoops`/`scanType`, +> `indexCosts`. The **top stage counts can look fine during a full scan** — drill +> into the nested `inputStage` down to the `COLLSCAN`/`IXSCAN`. See the +> `documentdb-query-performance-tuning` skill's +> `references/documentdb-explain-output.md` for the full field reference. + - **metrics**: `totalKeysExamined`, `totalDocsExamined`, `nReturned`, `executionTimeMillis` - **plan_shape**: winning plan stage (IXSCAN vs COLLSCAN), index used diff --git a/skills/query-optimizer/references/query-explain-plan.md b/skills/query-optimizer/references/query-explain-plan.md new file mode 100644 index 0000000..cbfa0bc --- /dev/null +++ b/skills/query-optimizer/references/query-explain-plan.md @@ -0,0 +1,58 @@ +# Verify index usage with `explain()` — the first check + +> Reference doc for the `documentdb-query-optimizer` skill (merged from the former +> `documentdb-query-optimization` rule). Load this to verify — **before** changing +> anything — whether a query actually uses an index or is doing a `COLLSCAN`. + +## Why it matters + +Guessing query performance is unreliable. `explain("executionStats")` is the +ground truth: it reveals whether a query used an index (`IXSCAN`), did a full +collection scan (`COLLSCAN`), was targeted to a single shard, or was +scatter-gather. Run it as the **first** step of any optimization — and as part of +PR review for any new hot-path query — before recommending an index. + +## Read the plan first — then beware the confounders + +`explain()` is where you start, but three signals in it routinely mislead. Check +the plan **first**, and do not trust these at face value: + +1. **The top-line numbers lie.** Azure DocumentDB's planner is PostgreSQL-backed, + so the top-level `executionTimeMillis` and `totalDocsExamined` are *post-sort / + post-merge* and can look healthy even during a full scan. **Drill into the + nested `inputStage` down to the `COLLSCAN`/`IXSCAN`** and read *its* + `totalDocsExamined` / `totalKeysExamined`. A query that returns in a few + milliseconds can still be scanning the whole collection. +2. **An index existing ≠ the index being used.** Check `winningPlan.indexName`. + The planner may ignore an index that isn't selective for this query shape, so + "there's already an index" does not mean the query is fine. +3. **Scatter-gather masquerades as a slow query.** On a sharded collection, + `shards[]` hitting *every* shard is the real cost — not a missing index. Fix it + by including the shard key in the filter, not by adding another index. + +## Incorrect + +```javascript +// Ship it, hope it's fast in prod. +const results = await db.orders.find(filter).sort(sort).toArray(); +``` + +## Correct + +```javascript +const plan = await db.orders.find(filter).sort(sort).explain("executionStats"); +// Review, in order: +// 1. the SCAN stage (drill into inputStage): IXSCAN (good) vs COLLSCAN (bad) +// 2. scan-stage totalDocsExamined / totalKeysExamined vs nReturned (ratio ~1 is ideal) +// 3. a blocking SORT stage (sort not served by the index) +// 4. shards[]: all shards hit => scatter-gather, fix with the shard key in the filter +``` + +Automate it in CI with a small harness that asserts +`scan-stage totalDocsExamined / nReturned < threshold` for critical queries, so a +regression to `COLLSCAN` fails the build. + +## References + +- MongoDB [`explain()`](https://www.mongodb.com/docs/manual/reference/method/cursor.explain/) docs +- Companion: [`core-indexing-principles.md`](core-indexing-principles.md) · the `documentdb-query-performance-tuning` skill's `references/documentdb-explain-output.md` (full DocumentDB explain field glossary) diff --git a/skills/query-performance-tuning/SKILL.md b/skills/query-performance-tuning/SKILL.md new file mode 100644 index 0000000..74232ed --- /dev/null +++ b/skills/query-performance-tuning/SKILL.md @@ -0,0 +1,218 @@ +--- +name: documentdb-query-performance-tuning +description: >- + End-to-end query performance tuning methodology for Azure DocumentDB. Use when + the user asks how to tune query performance, read or interpret + `explain("executionStats")` output, understand the ESR (Equality-Sort-Range) + rule, eliminate a `COLLSCAN` or an in-memory sort, design a compound index for + a query shape, confirm a sort is index-backed, use covered queries, or find + slow queries in production via Diagnostic Logs / Log Analytics + (`VCoreMongoRequests`). Covers DocumentDB's PostgreSQL-backed explain format + (`PARALLEL_SORT_MERGE`, cost estimates, `runtimeFilterSet`, `indexUsage`, + `indexCosts`). For interactive MCP-driven optimization of one specific query, + use `documentdb-query-optimizer` instead. +license: MIT +--- + +# Query Performance Tuning — Azure DocumentDB + +Writing a query that *works* is easy; writing one that *scales* is not. As a +collection grows to millions of documents, a query that was instant in +development starts taking seconds — almost always because the database is doing +far more work than it needs to. A single well-designed index can turn an +**81.3 ms** query into a **0.053 ms** one with **no application code change**. + +This skill is the repeatable mental model: + +> **find the slow query → read `explain()` → apply ESR → confirm the sort is +> index-backed → (optionally) cover the query.** + +Source: [Query Performance Tuning Guide](https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/) (Azure DocumentDB devblog). + +## When to use this skill + +- "How do I tune / speed up this query?" (methodology, not a live MCP session) +- "How do I read / interpret `explain()` output on DocumentDB?" +- "What is the ESR rule / how do I order a compound index?" +- "Why is my query doing a `COLLSCAN` / an in-memory `SORT`?" +- "What is a covered query and when is it worth it?" +- "How do I find the slowest queries in production?" + +For hands-on optimization of one specific query against a live cluster (running +`explain_operation`, proposing and creating an index via MCP), use the +**`documentdb-query-optimizer`** skill. This skill is the underlying theory both +share. + +## Step 0 — Find the slow queries first + +Before running `explain()`, know *which* query to investigate. On Azure +DocumentDB this uses **Diagnostic Logs** routed to an **Azure Log Analytics** +workspace (see the `documentdb-monitoring` skill to enable diagnostic settings). +Then rank the slowest operations with KQL: + +```kusto +VCoreMongoRequests +| where DurationMs > 1000 +| project TimeGenerated, DatabaseName, CollectionName, + OperationName, DurationMs, PiiCommandText +| order by DurationMs desc +| take 20 +``` + +`DurationMs` = execution time; `OperationName` = `find` / `aggregate` / …; +`PiiCommandText` = the actual command. Copy a candidate and run +`explain("executionStats")` on it. + +## The problem query + +```javascript +db.orders.find({ + status: "shipped", + customerId: "C-4821", + createdAt: { $gte: ISODate("2024-01-01") } +}).sort({ createdAt: -1 }) +``` + +Innocent-looking, but on a 1,000,000-document collection it becomes the worst +offender under load. The five steps below fix it. + +## Step 1 — Run `explain()` and read the scan stage + +```javascript +db.orders.find({ /* … */ }).sort({ createdAt: -1 }).explain("executionStats") +``` + +The damning part is the innermost stage: + +```json +"stage": "COLLSCAN", +"totalDocsExamined": 333333, +"totalDocsRemovedByRuntimeFilter": 333327, +"nReturned": 6 +``` + +**333,333 documents scanned to return 18** (across parallel workers) — a ~99.99% +waste. `COLLSCAN` = no index used. + +> **Read the scan stage, not the top stage.** Azure DocumentDB's planner is +> PostgreSQL-backed, so the top `PARALLEL_SORT_MERGE` stage reports *post-merge* +> counts that can look tiny (`totalDocsExamined: 18`) even during a full scan. +> Always drill into `inputStage` down to the `COLLSCAN`/`IXSCAN`. For the full +> DocumentDB explain field glossary, load +> [`references/documentdb-explain-output.md`](references/documentdb-explain-output.md). + +## Step 2 — A naive single-field index is not enough + +```javascript +db.orders.createIndex({ status: 1 }) +``` + +Now it's an `IXSCAN`, but still **249,800 index keys examined to return 18** — a +low-cardinality field like `status` narrows nothing useful, doesn't help +`customerId` / `createdAt`, and the sort still runs **in memory** +(`scanType: "regular"`, a `SORT` stage above the `IXSCAN`). + +## Step 3 — Apply the ESR rule + +> **E**quality → **S**ort → **R**ange + +| Role | Field(s) | +|---|---| +| **Equality** (`$eq`) | `customerId`, `status` | +| **Sort** | `createdAt` (`-1`) | +| **Range** (`$gte`) | `createdAt` | + +`createdAt` is used in both sort and range, so it appears **once**, in the sort +position (walking the index in sort order also satisfies the range). + +```javascript +db.orders.createIndex({ customerId: 1, status: 1, createdAt: -1 }) +``` + +**Most selective equality field first.** `customerId` (high cardinality: +thousands of distinct values) skips far more entries than `status` (low +cardinality: `shipped`/`pending`/`cancelled`), so it leads even though it isn't +unique. + +Re-run `explain()` → **18 keys examined, 18 returned, 0.058 ms, zero in-memory +sort** (`scanType: "ordered"`, `hasOrderBy: true`, a `FETCH → IXSCAN` plan). + +## Step 4 — Confirm the sort is index-backed + +Look for the **absence of a `SORT` stage**. Ideal: + +```text +FETCH +└── IXSCAN { customerId: 1, status: 1, createdAt: -1 } +``` + +Not ideal (sort still in memory): + +```text +SORT +└── FETCH + └── IXSCAN { status: 1 } +``` + +On DocumentDB, also confirm the `IXSCAN` reports `scanType: "ordered"` and +`hasOrderBy: true`. + +## Step 5 — Go further with covered queries + +Even a perfect `IXSCAN` still does a `FETCH` (index → back to the collection for +the document). A **covered query** puts *every* field the query touches — +filter, sort, **and projection** — in the index, so the engine never reads the +documents (no `FETCH` stage): + +```javascript +db.orders.find( + { status: "shipped", customerId: "C-4821", createdAt: { $gte: ISODate("2024-01-01") } }, + { _id: 0, customerId: 1, status: 1, createdAt: 1 } // only indexed fields; exclude _id +).sort({ createdAt: -1 }).explain("executionStats") +// winning plan collapses to just IXSCAN — 0.053 ms +``` + +**Worth it when:** high-read collections where the same query runs thousands of +times/sec and the projected fields are a small subset of a wide document. If you +need the full document anyway, the `FETCH` is unavoidable — don't contort the +projection just to chase coverage. + +## Gotchas + +- **Top-line explain metrics lie.** `executionTimeMillis` and `totalDocsExamined` + at the *top* stage can look fine during a full scan — read the nested scan + stage (see Step 1). +- **Range fields first breaks ESR.** `createIndex({ createdAt: -1, customerId: 1, status: 1 })` + forces a wide date-range scan before the equality filters apply. Equality first. +- **Mismatched multi-field sort direction.** The sort must match the index fields + **and directions** exactly, **or** be their complete reverse. Given `{ a: 1, b: -1 }`: + `sort({ a: 1, b: -1 })` ✅ and `sort({ a: -1, b: 1 })` ✅; `sort({ a: 1, b: 1 })` + ⚠️ only `a` uses the index and `b` is sorted in memory. +- **Over-indexing.** Every index costs write throughput and storage. Index for + *actual* query patterns confirmed with `explain()`, not preemptively. + +## Quick reference: ESR checklist + +Before creating a compound index, answer three questions: + +1. Which fields use **exact match** (`$eq`)? → **first** (most selective first). +2. Which field is used in **`sort()`**? → **middle**, matching the sort direction. +3. Which fields use **range operators** (`$gt`, `$lt`, `$gte`, `$lte`, `$in`)? → **last**. + +## Summary + +| Lever | Effect | +|---|---| +| Diagnostic Logs + Log Analytics (`VCoreMongoRequests`) | Surface slow queries before they become incidents | +| Replace `COLLSCAN` with `IXSCAN` | Eliminate the full collection scan | +| Apply the ESR rule to a compound index | Narrow key scans to near-exact matches | +| Match index direction to the sort | Eliminate the in-memory `SORT` | +| Cover the query with a projection | Eliminate the `FETCH` stage entirely | + +## References + +- Full DocumentDB explain field glossary + annotated plans: [`references/documentdb-explain-output.md`](references/documentdb-explain-output.md) +- [Query Performance Tuning Guide](https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/) (devblog) +- [How to read explain output](https://learn.microsoft.com/en-us/azure/documentdb/how-to-read-explain-output) +- [How to monitor diagnostics logs](https://learn.microsoft.com/en-us/azure/documentdb/how-to-monitor-diagnostics-logs) +- Related skills: `documentdb-query-optimizer` (interactive MCP tuning), `documentdb-indexing` (index-type selection), `documentdb-monitoring` (enable diagnostic settings) diff --git a/skills/query-performance-tuning/references/documentdb-explain-output.md b/skills/query-performance-tuning/references/documentdb-explain-output.md new file mode 100644 index 0000000..b72b760 --- /dev/null +++ b/skills/query-performance-tuning/references/documentdb-explain-output.md @@ -0,0 +1,127 @@ +# Reading Azure DocumentDB `explain("executionStats")` + +> Reference doc for the `documentdb-query-performance-tuning` skill. Load this +> when you need to interpret a **real** Azure DocumentDB explain plan. +> +> Azure DocumentDB's query planner is **PostgreSQL-backed**, so +> `explain("executionStats")` emits `"explainVersion": 2` output that carries +> **planner cost estimates** and **runtime buffer stats** you will not see in +> community MongoDB. Learn these fields — the MongoDB-standard summary is not +> enough to read a real DocumentDB plan. +> +> Source: [How to read explain output](https://learn.microsoft.com/en-us/azure/documentdb/how-to-read-explain-output), +> [Query Performance Tuning Guide](https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/) + +## DocumentDB-specific fields (not in community MongoDB) + +| Field | Where | What it means | +|---|---|---| +| `PARALLEL_SORT_MERGE` | winning-plan `stage` | Top stage that merges results from parallel workers (`workersPlanned` / `parallelWorkers`). Its own `totalDocsExamined` is the **merged** count and can look tiny — always drill into `inputStage`. | +| `startupCost` / `totalCost` | every stage | PostgreSQL planner cost estimate (not milliseconds). A huge `totalCost` on a `COLLSCAN` (e.g. `34418`) is a red flag even when wall-clock looks OK. | +| `estimatedTotalKeysExamined` | queryPlanner stages | Planner **estimate** of keys examined. Compare against the **actual** `totalKeysExamined` in `executionStats`. | +| `runtimeFilterSet` / `totalDocsRemovedByRuntimeFilter` | `COLLSCAN` | Filters applied *while scanning*. A large `totalDocsRemovedByRuntimeFilter` means the scan read and threw away almost everything. | +| `numBlocksFromCache` | execution stages | Buffer-cache blocks touched. A large value on a scan stage signals heavy page churn. | +| `indexUsage.scanLoops` | `IXSCAN` | Number of index key lookups performed. | +| `indexUsage.scanType` | `IXSCAN` | `"regular"` = scan without an ordering guarantee (an in-memory `SORT` may follow); `"ordered"` = index walked **in sort order** (no in-memory sort). | +| `indexUsage.scanKeys[].estimatedEntryCount` | `IXSCAN` | Estimated matching entries per key; `isInequality` flags a range bound. | +| `hasOrderBy` / `bounds` / `indexFilterSet` / `direction` | `IXSCAN` | `hasOrderBy: true` + a `direction` means the sort is served by the index. `bounds` shows the exact scanned key range. | +| `indexCosts[]` | queryPlanner | Per-candidate-index costing: `selectivity`, `correlation`, `estimatedPercentIndexPagesLoaded`, `estimatedTotalIndexEntries`, `boundarySelectivity`. | + +## Gotcha: the top-line numbers lie + +The **top stage** of a plan reports post-merge / post-sort counts, so +`totalDocsExamined` and `executionTimeMillis` at the top can look *great* even +during a full scan. You must drill into the nested `inputStage` chain down to the +`COLLSCAN` / `IXSCAN` to see the real work. In the worked example the top +`PARALLEL_SORT_MERGE` reports `totalDocsExamined: 18`, but the underlying +`COLLSCAN` reports `totalDocsExamined: 333333`. + +## The three plans, annotated + +### 1. `COLLSCAN` — no index (bad) + +```json +"winningPlan": { + "stage": "PARALLEL_SORT_MERGE", + "totalCost": 40692.13, + "inputStage": { + "stage": "SORT", + "sortKey": [ { "createdAt": -1 } ], + "inputStage": { + "stage": "COLLSCAN", + "totalCost": 34418.4, + "runtimeFilterSet": [ { "status": { "$eq": "shipped" } }, ... ] + } + } +} +``` +```json +"executionStats": { + "executionStages": { + "inputStage": { // drill down to the COLLSCAN + "inputStage": { + "stage": "COLLSCAN", + "totalDocsExamined": 333333, + "totalDocsRemovedByRuntimeFilter": 333327, + "numBlocksFromCache": 25000 + } + } + } +} +``` +Signals: `COLLSCAN` stage, huge `totalCost`, huge `totalDocsExamined` / +`totalDocsRemovedByRuntimeFilter` at the scan stage, blocking `SORT` above it. + +### 2. Naive single-field `IXSCAN` (better, still wrong) + +```json +"inputStage": { + "stage": "IXSCAN", + "indexName": "status_1", + "nReturned": 249800, + "totalKeysExamined": 249800, + "indexUsage": { "scanLoops": 249800, "scanType": "regular" } +} +``` +Signals: `IXSCAN` now, but `totalKeysExamined` is still enormous vs `nReturned`, +`scanType: "regular"` (not `"ordered"`), and a `SORT` stage still sits above it. + +### 3. ESR compound `IXSCAN` (good) + +```json +"winningPlan": { + "stage": "FETCH", + "inputStage": { + "stage": "IXSCAN", + "indexName": "customerId_1_status_1_createdAt_-1", + "direction": "Forward", + "hasOrderBy": true, + "indexUsage": { "scanLoops": 19, "scanType": "ordered" } + } +}, +"executionStats": { + "nReturned": 18, + "totalKeysExamined": 18, + "executionTimeMillis": 0.058 +} +``` +Signals: no `SORT` stage, `scanType: "ordered"`, `hasOrderBy: true`, +`totalKeysExamined ≈ nReturned`. A **covered** query removes even the `FETCH`, +leaving a bare `IXSCAN`. + +## What "good" looks like + +| Metric | Good | Bad | +|---|---|---| +| Scan-stage `stage` | `IXSCAN` | `COLLSCAN` | +| `totalKeysExamined / nReturned` | ≈ 1 | ≫ 1 (poor selectivity) | +| `totalDocsExamined / nReturned` (scan stage) | ≈ 1 | ≫ 1 (scanning too much) | +| `indexUsage.scanType` | `ordered` (when sorting) | `regular` + a `SORT` stage | +| `SORT` stage present | absent | present (blocking in-memory sort) | +| `FETCH` stage present | absent = covered query | present (unavoidable if you need the full doc) | + +## References + +- [How to read explain output](https://learn.microsoft.com/en-us/azure/documentdb/how-to-read-explain-output) +- [Query Performance Tuning Guide](https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/) (devblog) +- Companion: [`../../query-optimizer/references/core-indexing-principles.md`](../../query-optimizer/references/core-indexing-principles.md) · [`../../indexing/index-compound-esr.md`](../../indexing/index-compound-esr.md) diff --git a/skills/sharding/SKILL.md b/skills/sharding/SKILL.md index 30aa2db..e52b23d 100644 --- a/skills/sharding/SKILL.md +++ b/skills/sharding/SKILL.md @@ -38,4 +38,4 @@ collection's expected size or throughput ≤ one physical shard's budget? - [Sharding for horizontal scalability in Azure DocumentDB](https://learn.microsoft.com/azure/documentdb/partitioning) - [Compute and storage in Azure DocumentDB](https://learn.microsoft.com/azure/documentdb/compute-storage) -- Related: `indexing/` (index on the shard key), `query-optimization/` (single-shard vs scatter-gather queries), `high-availability/` (replica sets within each physical shard) +- Related: `indexing/` (index on the shard key), `query-optimizer/` (single-shard vs scatter-gather queries), `high-availability/` (replica sets within each physical shard) diff --git a/skills/sharding/sharding-hot-partition-diagnosis.md b/skills/sharding/sharding-hot-partition-diagnosis.md index 55edd97..50c1529 100644 --- a/skills/sharding/sharding-hot-partition-diagnosis.md +++ b/skills/sharding/sharding-hot-partition-diagnosis.md @@ -37,7 +37,7 @@ If you find a hot shard, work back through this list to find which shape applies Before resharding, rule out cheaper causes: -1. **Is one query pattern doing collection scans?** A single bad query can saturate one shard if it happens to route there. Run query stats / `explain()` on the heaviest queries. See `query-optimization/`. +1. **Is one query pattern doing collection scans?** A single bad query can saturate one shard if it happens to route there. Run query stats / `explain()` on the heaviest queries. See `query-optimizer/`. 2. **Is the index missing on the shard key?** If queries that include the shard key still scatter-gather, the index didn't get created (or got dropped). See [sharding-how-to-commands](sharding-how-to-commands.md). 3. **Is a background job concentrated on one shard's data?** E.g., a nightly export filtering on one tenant. Reschedule or batch differently before resharding. 4. **Confirm the metric is sustained, not a spike.** A 10-minute hot shard from a backup job isn't worth a reshard. A multi-day pattern is. diff --git a/skills/sharding/sharding-when-to-shard.md b/skills/sharding/sharding-when-to-shard.md index cd6c200..ddeaffc 100644 --- a/skills/sharding/sharding-when-to-shard.md +++ b/skills/sharding/sharding-when-to-shard.md @@ -51,7 +51,7 @@ Before sharding, confirm a properly-sized **single-shard** cluster cannot meet r - Project peak storage at the workload's 12–24 month horizon. Compare against the largest available storage SKU (currently up to 32 TB per shard). If the projection fits with headroom, stay single-shard. - Run the workload (or a representative load test) against the largest tier you'd consider. Measure sustained CPU, memory, IOPS, and request latency. If headroom remains, stay single-shard. -- Confirm indexing is correct (see `indexing/`) and queries are not doing accidental collection scans (see `query-optimization/`). These problems look like "we need to shard" but aren't. +- Confirm indexing is correct (see `indexing/`) and queries are not doing accidental collection scans (see `query-optimizer/`). These problems look like "we need to shard" but aren't. ### 2. Shard the right collections, not all of them diff --git a/testing/scenarios/kb-router/expected-findings.yaml b/testing/scenarios/kb-router/expected-findings.yaml index d8eba8d..7fdb330 100644 --- a/testing/scenarios/kb-router/expected-findings.yaml +++ b/testing/scenarios/kb-router/expected-findings.yaml @@ -35,6 +35,47 @@ routes: # Confident routes must clear this score (router sets "confident" at >= 2.0). min_confident_score: 2.0 +# Route B (skills): representative NL query -> skill id the router must resolve to +# via engine.rank_skills. Guards that promoting/adding a standalone skill keeps it +# (and its siblings) triggerable without colliding. +skill_routes: + - query: "how do I read the explain output and apply the ESR rule" + skill: query-performance-tuning + - query: "how do I find slow queries in production with log analytics" + skill: query-performance-tuning + - query: "how do I stop my query from doing a collection scan" + skill: query-performance-tuning + - query: "optimize this query and recommend an index" + skill: query-optimizer + - query: "how should I index this specific query" + skill: query-optimizer + - query: "which index type should I use for this query" + skill: indexing + - query: "which compound index should I design for this filter, sort, and range query" + skill: indexing + - query: "use explain to check whether this query uses an index or does a collection scan" + skill: query-optimizer + - query: "verify that my query is using an index and not scanning the collection" + skill: query-optimizer + - query: "help me write an aggregation pipeline to group by status" + skill: natural-language-querying + - query: "translate this SQL query into a MongoDB query" + skill: natural-language-querying + - query: "how do I set up the documentdb mcp server" + skill: mcp-setup + - query: "provision an azure documentdb cluster with bicep" + skill: azure-deployment + - query: "how do I tune my connection pool size and maxPoolSize" + skill: connection + - query: "I'm getting connection pool exhaustion and timeouts" + skill: connection + +# A skill query that must resolve to a skill AND be the recommended route +# (higher score than any script) end-to-end through the shell wrapper. +skill_shell_case: + query: "how do I read explain output and apply the ESR rule" + skill: query-performance-tuning + # A vague query with no tool-specific signal must NOT produce a confident route — # the router should decline rather than guess. unconfident_case: diff --git a/testing/scenarios/kb-router/tests/test_kb_router.py b/testing/scenarios/kb-router/tests/test_kb_router.py index fd651ef..0836f4a 100644 --- a/testing/scenarios/kb-router/tests/test_kb_router.py +++ b/testing/scenarios/kb-router/tests/test_kb_router.py @@ -52,6 +52,13 @@ def _route_cases(): return [(r["query"], r["tool"]) for r in spec["routes"]] +def _skill_route_cases(): + import yaml + spec = yaml.safe_load((Path(__file__).resolve().parents[1] / + "expected-findings.yaml").read_text()) + return [(r["query"], r["skill"]) for r in spec.get("skill_routes", [])] + + @pytest.mark.kbrouter @pytest.mark.parametrize("query,expected_tool", _route_cases()) def test_query_routes_to_expected_tool(engine, kb, query, expected_tool): @@ -73,6 +80,38 @@ def test_confident_routes_clear_threshold(engine, kb, expected): ) +# ── Route B (skills) routing correctness ──────────────────────────────────── +@pytest.mark.kbrouter +@pytest.mark.parametrize("query,expected_skill", _skill_route_cases()) +def test_query_routes_to_expected_skill(engine, kb, query, expected_skill): + ranked = engine.rank_skills(kb, query) + assert ranked, "rank_skills returned no skills — is kb.json 'skills[]' populated?" + score, skill, hits = ranked[0] + assert skill["id"] == expected_skill, ( + f"'{query}' routed to skill '{skill['id']}' (score {score:.2f}), " + f"expected '{expected_skill}'" + ) + + +@pytest.mark.kbrouter +def test_confident_skill_routes_clear_threshold(engine, kb, expected): + thr = expected["min_confident_score"] + for query, _ in _skill_route_cases(): + score, skill, _ = engine.rank_skills(kb, query)[0] + assert score >= thr, ( + f"'{query}' scored {score:.2f} for skill '{skill['id']}', " + f"below confident threshold {thr}" + ) + + +@pytest.mark.kbrouter +def test_every_registered_skill_points_at_an_existing_skill_md(kb): + """Each routable skill's `path` must resolve to a real SKILL.md on disk.""" + for sk in kb.get("skills", []): + p = kit.REPO_DIR / sk["path"] + assert p.is_file(), f"skill '{sk['id']}' path does not exist: {sk['path']}" + + # ── scoring transparency: the multiword-phrase (+3.0) rule ────────────────── @pytest.mark.kbrouter def test_multiword_phrase_matches_and_scores(engine, kb, expected): @@ -119,3 +158,35 @@ def test_shell_wrapper_emits_valid_json(engine, kb): assert data["match"]["tool"] == "document-bloat-advisor" assert data["confident"] is True assert data["match"]["command"].startswith("bash scripts/document-bloat-advisor.sh") + + +@pytest.mark.kbrouter +def test_shell_wrapper_routes_skill_query(engine, kb, expected): + """A skill-flavoured query must resolve to a skill and be the recommended + route end-to-end through the shell wrapper.""" + case = expected["skill_shell_case"] + p = subprocess.run( + ["bash", str(KB_SH), "--json", case["query"]], + capture_output=True, text=True, timeout=30, + ) + assert p.returncode == 0, f"kb-route.sh failed: {p.stderr}" + data = json.loads(p.stdout) + assert data["skill_match"] is not None, "expected a skill_match in the output" + assert data["skill_match"]["skill"] == case["skill"] + assert data["skill_match"]["open"].endswith("SKILL.md") + assert data["skill_confident"] is True + assert data["recommended"] == "skill" + + +@pytest.mark.kbrouter +def test_shell_wrapper_lists_skills(engine, kb): + """--skills --json emits the registered skill catalog as valid JSON.""" + p = subprocess.run( + ["bash", str(KB_SH), "--skills", "--json"], + capture_output=True, text=True, timeout=30, + ) + assert p.returncode == 0, f"kb-route.sh --skills failed: {p.stderr}" + data = json.loads(p.stdout) + ids = {s["id"] for s in data} + assert "query-performance-tuning" in ids + assert len(data) == len(kb.get("skills", [])) From b50b44d68e78c49193525926e20f508024a2d6eb Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Tue, 4 Aug 2026 18:07:09 +0000 Subject: [PATCH 09/13] Route tools with 1-gram keyword matching (recall over precision) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-word (bi/tri-gram) phrase matching for the diagnostic scripts is brittle: it requires the exact phrase as a substring, so paraphrases miss. Since the scripts are read-only, over-triggering is harmless — so favor recall. - `score_tool` gains an `onegram` mode: split each keyword into single tokens and score +1.5 per distinct matching query token. `rank_tools` uses it (Route A). - Skills (Route B) keep phrase-aware matching (`+3.0` full-phrase / `+1.5` token) — precision matters there, since routing to the wrong guidance is a real cost. - Tests: phrase-rule check moved to skills (`skill_phrase_case`); new `tool_onegram_case` guards the 1-gram token behavior; the "decline vague query" guarantee now asserts on skills (tools intentionally over-trigger). SCENARIO.md and knowledge-base/README.md document the tools=1-gram / skills=phrase split. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 86722878-7b4c-4d57-933c-3d4188317703 --- knowledge-base/README.md | 19 ++++- knowledge-base/kb_route.py | 74 ++++++++++++------- testing/scenarios/kb-router/SCENARIO.md | 20 +++-- .../kb-router/expected-findings.yaml | 27 +++++-- .../kb-router/tests/test_kb_router.py | 40 +++++++--- 5 files changed, 125 insertions(+), 55 deletions(-) diff --git a/knowledge-base/README.md b/knowledge-base/README.md index 45cf3bc..971c66f 100644 --- a/knowledge-base/README.md +++ b/knowledge-base/README.md @@ -96,12 +96,23 @@ Currently routed skills (Route B — text guidance): | `connection` | "tune my connection pool", "maxPoolSize for serverless", "pool exhaustion" | **Routing is deterministic:** the router scores the query against each target's -`keywords` / `example_queries` in `kb.json` (multi-word phrases weigh more than -single tokens) plus the `routes_one_hop` / `routes_one_hop_skills` signal, and -returns a ranked result **per space** (best script, best skill) with a confidence, +`keywords` / `example_queries` in `kb.json`, plus the `routes_one_hop` / +`routes_one_hop_skills` signal, and returns a ranked result **per space** (best +script, best skill) with a confidence, alternatives, and a `recommended` route. No LLM required; same input → same route. -## Multi-hop workflows (schema + scaffold) +**Keyword matching differs by space, on purpose:** + +- **Tools (Route A / scripts) — 1-gram:** each keyword is split into single + tokens; every distinct matching query token scores `+1.5`. This favors + **recall** — a paraphrase like *"scan of the whole collection"* still matches + the `collection scan` keyword. Harmless, because the scripts are read-only, so + triggering an extra diagnostic costs nothing. +- **Skills (Route B) — phrase-aware:** a multi-word keyword scores `+3.0` only as + a full-phrase substring (a single-word keyword `+1.5`). This keeps + **precision**, because routing to the wrong *guidance* is a real cost. + +Both spaces then add `+2.5×` best example-query overlap and `+2.0×` one-hop boost. Troubleshooting is rarely one script. The KB models a workflow as a **guarded diagnostic graph** (an AND/OR decision graph — the classic sequential-diagnosis diff --git a/knowledge-base/kb_route.py b/knowledge-base/kb_route.py index a8eb821..889b516 100644 --- a/knowledge-base/kb_route.py +++ b/knowledge-base/kb_route.py @@ -44,27 +44,45 @@ def fill(invocation, db, placeholder): return invocation -def score_tool(t, q_tokens, q_lower): - """Score one tool against the query. - - Signals (additive): - * multiword keyword phrase present as a substring -> +3.0 each - * single-word keyword present as a query token -> +1.5 each - * best example-query token overlap -> +2.5 * fraction - Returns (score, matched_keywords). +def score_tool(t, q_tokens, q_lower, onegram=False): + """Score one tool/skill against the query. + + Two keyword-matching modes: + + * ``onegram=False`` (phrase-aware, used for SKILLS): a multi-word keyword + scores +3.0 only when the whole phrase is present as a substring; a + single-word keyword scores +1.5 when present as a query token. Higher + precision — favored where picking the *right* guidance matters. + * ``onegram=True`` (1-gram, used for TOOLS/scripts): every keyword is split + into single-word tokens and each *distinct* query token that hits that set + scores +1.5. Higher recall (a paraphrase like "scan the collection" still + matches "collection scan"); harmless over-triggering since the diagnostic + scripts are read-only. + + Both modes then add the best example-query token overlap (+2.5 * fraction). + Returns (score, matched_keywords) — for 1-gram, matched_keywords are the + matched single tokens. """ score = 0.0 hits = [] - for kw in t.get("keywords", []): - kwl = kw.lower() - if " " in kwl: - if kwl in q_lower: - score += 3.0 - hits.append(kw) - else: - if kwl in q_tokens: - score += 1.5 - hits.append(kw) + if onegram: + kw_tokens = set() + for kw in t.get("keywords", []): + kw_tokens |= (tokenize(kw) - STOP) + for tok in sorted(q_tokens & kw_tokens): + score += 1.5 + hits.append(tok) + else: + for kw in t.get("keywords", []): + kwl = kw.lower() + if " " in kwl: + if kwl in q_lower: + score += 3.0 + hits.append(kw) + else: + if kwl in q_tokens: + score += 1.5 + hits.append(kw) best_ex = 0.0 for ex in t.get("example_queries", []): ex_tokens = tokenize(ex) - STOP @@ -76,9 +94,10 @@ def score_tool(t, q_tokens, q_lower): def rank_tools(kb, query): - """Rank all tools (Route A scripts) for a query. Returns (ranked, q_tokens, - q_lower) where ranked is a list of (score, tool, matched_keywords) sorted - best-first.""" + """Rank all tools (Route A scripts) for a query. Tools use **1-gram** + keyword matching (recall-favoring; harmless over-triggering as scripts are + read-only). Returns (ranked, q_tokens, q_lower) where ranked is a list of + (score, tool, matched_tokens) sorted best-first.""" q_tokens = tokenize(query) - STOP q_lower = query.lower() @@ -93,7 +112,7 @@ def rank_tools(kb, query): ranked = [] for t in kb["tools"]: - s, hits = score_tool(t, q_tokens, q_lower) + s, hits = score_tool(t, q_tokens, q_lower, onegram=True) s += 2.0 * route_boost.get(t["id"], 0.0) ranked.append((s, t, hits)) ranked.sort(key=lambda x: -x[0]) @@ -101,12 +120,11 @@ def rank_tools(kb, query): def rank_skills(kb, query): - """Rank all skills (Route B text targets) for a query, mirroring rank_tools. - - Uses the same additive scoring (score_tool) over each skill's - keywords/example_queries, plus a routes_one_hop_skills boost. Returns a list - of (score, skill, matched_keywords) sorted best-first, or [] if the KB - defines no skills.""" + """Rank all skills (Route B text targets) for a query. Skills keep the + **phrase-aware** matching (score_tool default) — higher precision, because + routing to the wrong *guidance* is a real cost (unlike an extra read-only + script). Returns a list of (score, skill, matched_keywords) sorted + best-first, or [] if the KB defines no skills.""" skills = kb.get("skills", []) if not skills: return [] diff --git a/testing/scenarios/kb-router/SCENARIO.md b/testing/scenarios/kb-router/SCENARIO.md index 363c87d..261a682 100644 --- a/testing/scenarios/kb-router/SCENARIO.md +++ b/testing/scenarios/kb-router/SCENARIO.md @@ -14,10 +14,16 @@ scenario protects two things: 1. **Routing correctness** — a set of representative questions must resolve to the expected tool id (e.g. TOAST/large-document questions → `document-bloat-advisor`). -2. **Scoring transparency** — the additive rules that produce the score - (`+3.0` multiword phrase, `+1.5` single keyword, `+2.5×` example overlap, - `+2.0×` one-hop boost) stay intact, and confident routes clear the `>= 2.0` - threshold. +2. **Scoring transparency** — the additive rules that produce the score stay + intact, and confident routes clear the `>= 2.0` threshold. Keyword matching + differs by target space: + - **Tools (Route A / scripts): 1-gram** — every keyword is split into single + tokens; each distinct matching query token scores `+1.5`. Recall-favoring + (harmless over-triggering, since scripts are read-only). + - **Skills (Route B): phrase-aware** — a multi-word keyword scores `+3.0` + only as a full-phrase substring, a single-word keyword `+1.5`. Precision is + kept because routing to the wrong guidance is a real cost. + - Both spaces then add `+2.5×` example overlap and `+2.0×` one-hop boost. It also runs the `kb-route.sh` wrapper end-to-end via subprocess to guard the shell → `kb_route.py` seam (env-var handoff, valid `--json`). @@ -30,9 +36,11 @@ Nothing — the router reads `knowledge-base/kb.json` (the shipped KB). There is ## Contract [`expected-findings.yaml`](expected-findings.yaml): -- `routes:` — a list of `{query, tool}` pairs the router must satisfy. +- `routes:` / `skill_routes:` — `{query, tool}` / `{query, skill}` pairs the router must satisfy. - `min_confident_score` — confident routes must score at least this. -- `phrase_case` — a query that must match a specific multiword keyword. +- `skill_phrase_case` — a query matching a multi-word **skill** keyword (skills keep phrase matching). +- `tool_onegram_case` — a query whose multi-word **tool** keyword is matched via its single tokens (tools are 1-gram). +- `unconfident_case` — a vague query that must decline for **skills** (tools intentionally over-trigger). ## Run diff --git a/testing/scenarios/kb-router/expected-findings.yaml b/testing/scenarios/kb-router/expected-findings.yaml index 7fdb330..cd560a9 100644 --- a/testing/scenarios/kb-router/expected-findings.yaml +++ b/testing/scenarios/kb-router/expected-findings.yaml @@ -76,13 +76,26 @@ skill_shell_case: query: "how do I read explain output and apply the ESR rule" skill: query-performance-tuning -# A vague query with no tool-specific signal must NOT produce a confident route — -# the router should decline rather than guess. +# A vague, non-specific query. TOOLS intentionally over-trigger under 1-gram +# matching (running an extra read-only script is harmless), so the "decline +# rather than guess" guarantee is asserted on SKILLS, where routing to the wrong +# guidance is a real cost. This query must stay below the confident threshold +# for skills (engine.rank_skills). unconfident_case: query: "my scans read tons of data even though the filter is simple" -# A query that must match a specific MULTIWORD keyword (the +3.0 phrase rule). -phrase_case: - query: "is large text hurting my scans" - tool: document-bloat-advisor - keyword: "large text" +# A query that must match a specific MULTIWORD keyword via the +3.0 phrase rule. +# Phrase matching now applies to SKILLS only (tools use 1-gram); this is checked +# with engine.rank_skills. +skill_phrase_case: + query: "how do I read the explain output" + skill: query-performance-tuning + keyword: "explain output" + +# A query with a multi-word TOOL keyword whose CONSTITUENT tokens must match +# under the tools' 1-gram rule (the phrase itself is not required as a hit). +tool_onegram_case: + query: "help me diagnose a collection scan problem" + tool: perf-advisor + token: "scan" + diff --git a/testing/scenarios/kb-router/tests/test_kb_router.py b/testing/scenarios/kb-router/tests/test_kb_router.py index 0836f4a..660b40d 100644 --- a/testing/scenarios/kb-router/tests/test_kb_router.py +++ b/testing/scenarios/kb-router/tests/test_kb_router.py @@ -113,18 +113,36 @@ def test_every_registered_skill_points_at_an_existing_skill_md(kb): # ── scoring transparency: the multiword-phrase (+3.0) rule ────────────────── +# ── scoring transparency: phrase rule (SKILLS) + 1-gram rule (TOOLS) ──────── @pytest.mark.kbrouter -def test_multiword_phrase_matches_and_scores(engine, kb, expected): - case = expected["phrase_case"] - score, tool, hits = _route(engine, kb, case["query"]) - assert tool["id"] == case["tool"] +def test_multiword_phrase_matches_and_scores_for_skills(engine, kb, expected): + """Skills keep phrase-aware matching: a multi-word skill keyword present as a + substring scores +3.0 and appears verbatim in the matched keywords.""" + case = expected["skill_phrase_case"] + score, skill, hits = engine.rank_skills(kb, case["query"])[0] + assert skill["id"] == case["skill"] assert case["keyword"] in hits, ( f"expected multiword keyword '{case['keyword']}' in matched hits {hits}" ) - # a single multiword phrase hit alone is worth +3.0 assert score >= 3.0, f"multiword phrase should score >= 3.0, got {score:.2f}" +@pytest.mark.kbrouter +def test_tool_onegram_matches_constituent_tokens(engine, kb, expected): + """Tools use 1-gram matching: a multi-word tool keyword is matched via its + constituent single tokens (the whole phrase is NOT required as a hit).""" + case = expected["tool_onegram_case"] + score, tool, hits = _route(engine, kb, case["query"]) + assert tool["id"] == case["tool"], ( + f"'{case['query']}' routed to '{tool['id']}', expected '{case['tool']}'" + ) + assert case["token"] in hits, ( + f"expected single token '{case['token']}' in matched hits {hits}" + ) + # hits are single tokens, never multi-word phrases, under 1-gram + assert all(" " not in h for h in hits), f"1-gram hits must be single tokens: {hits}" + + @pytest.mark.kbrouter def test_gibberish_is_not_confident(engine, kb): """A query matching nothing must not produce a confident route.""" @@ -134,14 +152,16 @@ def test_gibberish_is_not_confident(engine, kb): @pytest.mark.kbrouter def test_vague_query_declines_rather_than_guesses(engine, kb, expected): - """A vague query with no tool-specific signal must fall below the confident - threshold — the router should decline instead of over-confidently guessing.""" + """A vague, non-specific query must fall below the confident threshold for + SKILLS — where routing to the wrong guidance is a real cost, the router + declines instead of guessing. (Tools intentionally over-trigger under 1-gram + matching, since running an extra read-only script is harmless.)""" query = expected["unconfident_case"]["query"] - score, tool, _ = _route(engine, kb, query) + score, skill, _ = engine.rank_skills(kb, query)[0] thr = expected["min_confident_score"] assert score < thr, ( - f"vague query '{query}' should be below the confident threshold {thr}, " - f"but scored {score:.2f} for '{tool['id']}'" + f"vague query '{query}' should be below the confident threshold {thr} " + f"for skills, but scored {score:.2f} for '{skill['id']}'" ) From defbd3057740b46b1789bf4bba6e636a7dc3b176 Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 10 Aug 2026 20:25:00 +0000 Subject: [PATCH 10/13] quickstart: add the missing "open a Mongo shell" step + make numbers reproducible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The quickstart jumped straight into `db.orders.find(...)` after seeding without telling the reader how to run it. Add "Step 3 — Open a Mongo shell" (interactive `docker exec -it ... mongosh`) so every `db.orders.…` command has an environment, and run the later steps at that prompt instead of burying the connection in `--eval` one-liners. Steps renumbered 3->4, 4->5, 5->6. Also make the hero example honest: the ecommerce seed uses unseeded Math.random(), so a specific customer id / exact match count are not reproducible. Add a "pick the busiest customer" helper, anchor on the always-true 50,000-document scan, frame the after-count as "the handful that match (~10)", and note that exact numbers vary with the random sample. Verified end-to-end against a live container. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 86722878-7b4c-4d57-933c-3d4188317703 --- README.md | 2 +- docs/quickstart-find-and-fix-slow-queries.md | 128 ++++++++++++++----- 2 files changed, 98 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index e178950..cff1c08 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Skills follow the [Agent Skills](https://agentskills.io/) format and the kit shi 👉 **Capabilities and skill catalog:** [`docs/SKILLS.md`](docs/SKILLS.md) -👉 **New here? 10-minute quick start:** [Find & fix a slow query](docs/quickstart-find-and-fix-slow-queries.md) — run DocumentDB locally, load sample data, and let your AI assistant diagnose a `COLLSCAN` and fix it with one index (50,000 docs scanned → 10). +👉 **New here? 10-minute quick start:** [Find & fix a slow query](docs/quickstart-find-and-fix-slow-queries.md) — run DocumentDB locally, load sample data, and let your AI assistant diagnose a `COLLSCAN` and fix it with one index (a full 50,000-document scan → a tiny index lookup). ## Diagnostic Toolbox — Quickstart diff --git a/docs/quickstart-find-and-fix-slow-queries.md b/docs/quickstart-find-and-fix-slow-queries.md index 32503b9..ba0b51f 100644 --- a/docs/quickstart-find-and-fix-slow-queries.md +++ b/docs/quickstart-find-and-fix-slow-queries.md @@ -2,8 +2,9 @@ **Time: ~10 minutes.** In this guide you'll run Azure DocumentDB locally, load a sample store dataset, ask your AI coding assistant *why a query is slow*, apply the -one-line fix it recommends, and watch the same query go from scanning **50,000 -documents** to just **10** — with no application code change. +one-line fix it recommends, and watch the same query go from scanning **all 50,000 +documents** to a **tiny index lookup** of just the rows that match — with no +application code change. You don't need to know anything about the internals. If you have Docker and an AI coding assistant, you can follow along by copy-paste. @@ -32,7 +33,7 @@ git clone https://github.com/Azure/documentdb-agent-kit cd documentdb-agent-kit ``` -> **Preview:** Azure DocumentDB and this kit are in public preview; details may +> **Preview:** DocumentDB agent-kit is in public preview; details may > change. No production SLA. --- @@ -70,29 +71,73 @@ The `orders` collection starts with **no indexes** (other than the default `_id` --- -## Step 3 — Ask your assistant why a query is slow +## Step 3 — Open a Mongo shell -Here's a perfectly reasonable query — find a customer's shipped orders since the -start of the year, newest first: +The queries in the next steps are **MongoDB commands** — you run them at an +interactive `mongosh` prompt connected to your local database. Open one now: + +```bash +docker exec -it -u documentdb documentdb-local mongosh "localhost:10260/ecommerce" \ + -u docdbadmin -p Test1234 --authenticationMechanism SCRAM-SHA-256 --tls \ + --tlsAllowInvalidCertificates +``` + +You'll land at a prompt like this — every `db.orders.…` command below is typed here: + +```text +[direct: mongos] ecommerce> +``` + +> Type `exit` (or press `Ctrl-D`) to leave the shell. Already using the DocumentDB +> **MCP server** with your AI assistant? You can skip this shell — the assistant +> runs the same commands for you. + +--- + +## Step 4 — Ask your assistant why a query is slow + +Here's a perfectly reasonable query — find one customer's shipped orders since the +start of the year, newest first. + +The sample data is generated **randomly**, so first grab a customer that actually +has several shipped orders in *your* copy (run this at the `ecommerce>` prompt): + +```javascript +db.orders.aggregate([ + { $match: { status: "shipped", created_at: { $gte: ISODate("2024-01-01") } } }, + { $group: { _id: "$customer_id", n: { $sum: 1 } } }, + { $sort: { n: -1 } }, { $limit: 1 } +]) +``` + +That prints the busiest customer (for example `CUST_004087`). Use that id in the +query below: ```javascript db.orders.find({ status: "shipped", - customer_id: "CUST_004087", + customer_id: "CUST_004087", // ← use the id printed above created_at: { $gte: ISODate("2024-01-01") } }).sort({ created_at: -1 }) ``` -Paste this prompt to your AI assistant (the kit's skill will pick it up): +Paste this prompt to your AI assistant — swap in your customer id (the kit's skill +will pick it up): > **"Why is this query slow, and how do I fix it?"** > `db.orders.find({ status: "shipped", customer_id: "CUST_004087", created_at: { $gte: ISODate("2024-01-01") } }).sort({ created_at: -1 })` Your assistant will run `explain("executionStats")` and spot the problem: a **collection scan** (`COLLSCAN`) — the database reads **every one** of the 50,000 -documents to return just 10. - -> **Prefer to look yourself?** Run the same query with `.explain("executionStats")`. +documents just to return the handful (about 10) that match. + +> **Prefer to look yourself?** At the `ecommerce>` prompt from Step 3, run the +> same query with `.explain("executionStats")` appended: +> +> ```javascript +> db.orders.find({ status: "shipped", customer_id: "CUST_004087", created_at: { $gte: ISODate("2024-01-01") } }).sort({ created_at: -1 }).explain("executionStats") +> ``` +> > ⚠️ **Read the scan stage, not the summary.** The top-line time can look tiny (a > few milliseconds) even during a full scan — look at the `COLLSCAN` stage, where > `totalDocsExamined` is **50,000**. @@ -102,37 +147,56 @@ right skill for a question — `bash knowledge-base/kb-route.sh "why is this que --- -## Step 4 — Apply the recommended fix +## Step 5 — Apply the recommended fix The skill recommends a single **compound index**, ordered by the **ESR rule** -(Equality → Sort → Range), with the most selective field first: +(Equality → Sort → Range), with the most selective field first. At the +`ecommerce>` prompt, run: ```javascript db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 }) ``` -Create it (your assistant will ask for approval first): +(Your assistant will ask for approval before creating it.) -```bash -docker exec -u documentdb documentdb-local mongosh "localhost:10260/ecommerce" \ - -u docdbadmin -p Test1234 --authenticationMechanism SCRAM-SHA-256 --tls \ - --tlsAllowInvalidCertificates --quiet \ - --eval 'db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 })' -``` +> **Not in the shell?** You can run it from your host in one line instead: +> +> ```bash +> docker exec -u documentdb documentdb-local mongosh "localhost:10260/ecommerce" \ +> -u docdbadmin -p Test1234 --authenticationMechanism SCRAM-SHA-256 --tls \ +> --tlsAllowInvalidCertificates --quiet \ +> --eval 'db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 })' +> ``` -## Step 5 — See the improvement +## Step 6 — See the improvement -Run `explain("executionStats")` on the same query again. The `COLLSCAN` is gone, -replaced by an **index scan** (`IXSCAN`) that examines only the rows it needs: +Run the query with `.explain("executionStats")` again at the `ecommerce>` prompt: + +```javascript +db.orders.find({ + status: "shipped", + customer_id: "CUST_004087", + created_at: { $gte: ISODate("2024-01-01") } +}).sort({ created_at: -1 }).explain("executionStats") +``` + +The `COLLSCAN` is gone, replaced by an **index scan** (`IXSCAN`) that examines +only the rows it needs: | | Before (no index) | After (ESR index) | |---|---|---| | Plan | `COLLSCAN` (full scan) | `IXSCAN` (index scan) | -| Documents/keys examined | **50,000** | **10** | -| Rows returned | 10 | 10 | +| Documents/keys examined | **50,000** (every document) | **only the matching rows** (≈10) | +| Rows returned | ≈10 | ≈10 | + +**Thousands of times less work per query** (≈5,000× in the example above). Same +result, a fraction of the resources — which means more headroom to scale without +adding infrastructure. -**~5,000× less work per query.** Same result, a fraction of the resources — which -means more headroom to scale without adding infrastructure. +> **Your exact numbers will differ** — the sample data is generated randomly, so +> the number of matching orders (and which customer is busiest) changes each time +> you seed. What always holds is the pattern: a full **50,000-document scan** +> collapses to a **tiny index lookup** of just the rows that match. --- @@ -147,14 +211,16 @@ bash scenarios/ecommerce/query-perf-skill-test.sh ``` You'll see the same pattern across different query shapes — the size of the win -depends on how *selective* the filter is: +depends on how *selective* the filter is (the **before → after** figures below are +from one example run; your exact counts will vary with the random sample data, but +the `order_id` lookup is always a unique 50,000 → 1): | Ask your assistant… | Recommended index | Examined: before → after | |---|---|---| | "Does this query use an index or a collection scan?" (`find` by `order_id`) | `{ order_id: 1 }` | 50,000 → **1** | -| "Optimize `find({status, shipping_city}).sort({created_at})`" | `{ status: 1, shipping_city: 1, created_at: -1 }` | 50,000 → **998** | -| "Tune the flagship `find({status, customer_id, created_at}).sort()`" | `{ customer_id: 1, status: 1, created_at: -1 }` | 50,000 → **10** | -| "Which compound index for `find({status, total_amount:{$gt}}).sort({created_at})`?" | `{ status: 1, created_at: -1, total_amount: 1 }` | 50,000 → **8,899** | +| "Optimize `find({status, shipping_city}).sort({created_at})`" | `{ status: 1, shipping_city: 1, created_at: -1 }` | 50,000 → **~1,000** | +| "Tune the flagship `find({status, customer_id, created_at}).sort()`" | `{ customer_id: 1, status: 1, created_at: -1 }` | 50,000 → **~10** | +| "Which compound index for `find({status, total_amount:{$gt}}).sort({created_at})`?" | `{ status: 1, created_at: -1, total_amount: 1 }` | 50,000 → **~8,900** | *(The test creates each index, measures, and drops it again, so your `orders` collection is left exactly as it started.)* From 7ab1cc3c97da95a423bf056ce2a5220de9883ed6 Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 10 Aug 2026 20:27:29 +0000 Subject: [PATCH 11/13] quickstart: give the assistant prompt real context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Step 4 prompt was just a question plus a bare query, so the assistant had no idea which database was meant and no way to run explain() — it could only give generic advice. Replace it with a self-contained, copy-pasteable prompt that states the deployment (local documentdb-local container), database/collection, collection size, current indexes, and the orders document shape, plus a docker exec fallback so the assistant can run commands when no MCP connection exists. Add a short note explaining why the context matters and that MCP users can drop the exec line. Schema, status values, index state and the exec command all verified against the live container; the prompt still routes to the query-perf skill. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 86722878-7b4c-4d57-933c-3d4188317703 --- docs/quickstart-find-and-fix-slow-queries.md | 35 ++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/quickstart-find-and-fix-slow-queries.md b/docs/quickstart-find-and-fix-slow-queries.md index ba0b51f..444c095 100644 --- a/docs/quickstart-find-and-fix-slow-queries.md +++ b/docs/quickstart-find-and-fix-slow-queries.md @@ -122,10 +122,39 @@ db.orders.find({ ``` Paste this prompt to your AI assistant — swap in your customer id (the kit's skill -will pick it up): +will pick it up). Give it the surrounding context too, so it can inspect the +database itself instead of guessing: -> **"Why is this query slow, and how do I fix it?"** -> `db.orders.find({ status: "shipped", customer_id: "CUST_004087", created_at: { $gte: ISODate("2024-01-01") } }).sort({ created_at: -1 })` +```text +Why is this query slow on my Azure DocumentDB database, and how do I fix it? + + db.orders.find({ + status: "shipped", + customer_id: "CUST_004087", + created_at: { $gte: ISODate("2024-01-01") } + }).sort({ created_at: -1 }) + +Context: +- Azure DocumentDB (MongoDB-compatible) running locally in Docker, + container "documentdb-local", database "ecommerce", collection "orders". +- ~50,000 orders. The only index is the default _id — I haven't added any. +- Each order has: order_id, customer_id, status (pending/confirmed/shipped/ + delivered/cancelled), payment_method, created_at, updated_at, shipping_city, + total_amount. +- If you don't have a database connection, run commands through the shell: + docker exec -u documentdb documentdb-local mongosh "localhost:10260/ecommerce" \ + -u docdbadmin -p Test1234 --authenticationMechanism SCRAM-SHA-256 --tls \ + --tlsAllowInvalidCertificates --quiet --eval '' + +Please run explain("executionStats"), tell me what the plan is doing, and +recommend an index. Ask me before creating anything. +``` + +> **Why the extra context?** Without it the assistant has to guess which database +> you mean and has no way to run `explain()` — so it can only give generic advice. +> With it, it inspects *your* data and gives a specific answer. If you've set up +> the [DocumentDB MCP server](https://github.com/microsoft/documentdb-mcp), the +> assistant already has a connection and you can drop the `docker exec` line. Your assistant will run `explain("executionStats")` and spot the problem: a **collection scan** (`COLLSCAN`) — the database reads **every one** of the 50,000 From b6252665b2a6393159f0b39a25329c642a43a1ff Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 10 Aug 2026 22:01:46 +0000 Subject: [PATCH 12/13] Replace the hardcoded demo password with a DB_PASSWORD placeholder Every doc, seeder, and diagnostic script hardcoded the same literal demo password, which teaches a bad habit and trips secret scanners. Read it from the DB_PASSWORD environment variable instead: - Docs (quickstart, DIAGNOSTICS, README, scenario READMEs, the query-perf test write-up) export a placeholder once up front, start the container with -e PASSWORD="$DB_PASSWORD", and use -p "$DB_PASSWORD" in later commands. The export was moved above `docker run` so the variable exists when it's referenced. - Script/seeder usage comments and "no password" error hints, plus the pytest skip message, now suggest a placeholder rather than a real value. - The quickstart's AI-assistant prompt says the password is in DB_PASSWORD instead of embedding it. Verified end-to-end on a fresh container created with a non-default password: seed, interactive shell, explain (COLLSCAN docsExamined=50000), createIndex, IXSCAN, the before/after harness, a live perf-advisor run, bash -n on every edited script, and the kb-router suite (38 passed). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 86722878-7b4c-4d57-933c-3d4188317703 --- README.md | 4 +-- docs/DIAGNOSTICS.md | 10 ++++---- ...f-skill-test-instruction-20260730180441.md | 9 ++++--- docs/quickstart-find-and-fix-slow-queries.md | 25 +++++++++++++------ scenarios/contoso/README.md | 2 +- scenarios/contoso/seed.sh | 2 +- scenarios/ecommerce/README.md | 2 +- scenarios/ecommerce/query-perf-skill-test.sh | 4 +-- scenarios/ecommerce/seed.sh | 4 +-- scenarios/index-redundancy/README.md | 2 +- scenarios/index-redundancy/seed.sh | 4 +-- scripts/data-integrity-check.sh | 2 +- scripts/document-bloat-advisor.sh | 2 +- scripts/index-redundancy-finder.sh | 2 +- scripts/perf-advisor.sh | 2 +- scripts/toast-split-advisor.sh | 2 +- testing/conftest.py | 2 +- 17 files changed, 46 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index cff1c08..7ceb785 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,10 @@ is baked in** — set `DB_PASSWORD` (or pass `--password`). ```bash # 0. start a local DocumentDB container (choose any password; the scripts read it) +export DB_PASSWORD='' # the scripts require this (or --password) docker run -dt --name documentdb-local -p 10260:10260 \ - -e USERNAME=docdbadmin -e PASSWORD=Test1234 \ + -e USERNAME=docdbadmin -e PASSWORD="$DB_PASSWORD" \ ghcr.io/microsoft/documentdb/documentdb-local:latest -export DB_PASSWORD=Test1234 # the scripts require this (or --password) # 1. seed demo data bash scenarios/ecommerce/seed.sh # -> "ecommerce" diff --git a/docs/DIAGNOSTICS.md b/docs/DIAGNOSTICS.md index cd3cd2f..8cad3d1 100644 --- a/docs/DIAGNOSTICS.md +++ b/docs/DIAGNOSTICS.md @@ -26,15 +26,15 @@ password; the scripts read it from `DB_USER` (default `docdbadmin`) and `DB_PASSWORD` — **nothing is baked in**: ```bash +# the scripts require a password — export it once (or pass --password each time) +export DB_PASSWORD='' + docker run -dt --name documentdb-local \ -p 10260:10260 \ -e USERNAME=docdbadmin \ - -e PASSWORD=Test1234 \ + -e PASSWORD="$DB_PASSWORD" \ ghcr.io/microsoft/documentdb/documentdb-local:latest -# the scripts require a password — export it once (or pass --password each time) -export DB_PASSWORD=Test1234 - # preflight: confirm the engine answers (should print "1") docker exec documentdb-local psql -h localhost -p 9712 -U documentdb -d postgres -tAc "SELECT 1" ``` @@ -96,7 +96,7 @@ python3 knowledge-base/kb_route_demo.py "which indexes can I drop" # apply the schema split the advisor recommends, then re-run the advisor docker cp scenarios/contoso/contoso-split-fix.js documentdb-local:/tmp/fix.js docker exec -e CONTOSO_DB=contoso documentdb-local mongosh \ - "localhost:10260/contoso" -u docdbadmin -p Test1234 \ + "localhost:10260/contoso" -u docdbadmin -p "$DB_PASSWORD" \ --authenticationMechanism SCRAM-SHA-256 --tls --tlsAllowInvalidCertificates \ --quiet --file /tmp/fix.js bash scripts/document-bloat-advisor.sh --db contoso # opportunities now clean diff --git a/docs/query-perf-skill-test-instruction-20260730180441.md b/docs/query-perf-skill-test-instruction-20260730180441.md index c1dccd3..d142bd6 100644 --- a/docs/query-perf-skill-test-instruction-20260730180441.md +++ b/docs/query-perf-skill-test-instruction-20260730180441.md @@ -58,7 +58,7 @@ is identical. | Container name | `documentdb-local` | | Gateway (Mongo API) | container-internal `localhost:10260`, TLS, SCRAM-SHA-256 | | PostgreSQL engine | container-internal `localhost:9712` | -| Credentials | user `docdbadmin`, password `Test1234` (`DB_PASSWORD`) | +| Credentials | user `docdbadmin`, from the `DB_PASSWORD` env var (choose your own) | | mongosh client | `2.3.8` | > **Reproducibility gotcha:** this `documentdb-local` image **does not bundle a `mongosh` @@ -74,12 +74,13 @@ is identical. ### Bring up + seed ```bash +# 0. pick a password for the throwaway local container; everything below reads it +export DB_PASSWORD='' # 1. start the container (host ports optional; seeders use docker exec) -docker run -d --name documentdb-local -e USERNAME=docdbadmin -e PASSWORD=Test1234 \ +docker run -d --name documentdb-local -e USERNAME=docdbadmin -e PASSWORD="$DB_PASSWORD" \ ghcr.io/microsoft/documentdb/documentdb-local:latest # 2. install mongosh into the container (see gotcha above) # 3. seed the ecommerce dataset (~50K orders; takes ~1 min) -export DB_PASSWORD=Test1234 bash scenarios/ecommerce/seed.sh ``` @@ -298,7 +299,7 @@ matter and are scale-independent; absolute milliseconds are not comparable. ```bash # prerequisites: container up + mongosh installed + ecommerce seeded (see §2) -export DB_PASSWORD=Test1234 +export DB_PASSWORD='' bash scenarios/ecommerce/query-perf-skill-test.sh # prints the human report + a JSON block ``` diff --git a/docs/quickstart-find-and-fix-slow-queries.md b/docs/quickstart-find-and-fix-slow-queries.md index 444c095..a478f9b 100644 --- a/docs/quickstart-find-and-fix-slow-queries.md +++ b/docs/quickstart-find-and-fix-slow-queries.md @@ -40,9 +40,20 @@ cd documentdb-agent-kit ## Step 1 — Run DocumentDB locally +First pick a password for your local container. Every command in this guide reads +it from `DB_PASSWORD`, so you only type it once (choose anything — this container +is a throwaway on your own machine): + +```bash +export DB_PASSWORD='' +``` + +> Keep it in your shell for the whole walkthrough. If you open a **new terminal**, +> re-run that `export` before continuing. + ```bash docker run -d --name documentdb-local \ - -e USERNAME=docdbadmin -e PASSWORD=Test1234 \ + -e USERNAME=docdbadmin -e PASSWORD="$DB_PASSWORD" \ ghcr.io/microsoft/documentdb/documentdb-local:latest ``` @@ -62,7 +73,6 @@ This seeds a realistic store database (`ecommerce`) — **50,000 orders**, customers, products, and more. It takes about a minute. ```bash -export DB_PASSWORD=Test1234 bash scenarios/ecommerce/seed.sh ``` @@ -78,7 +88,7 @@ interactive `mongosh` prompt connected to your local database. Open one now: ```bash docker exec -it -u documentdb documentdb-local mongosh "localhost:10260/ecommerce" \ - -u docdbadmin -p Test1234 --authenticationMechanism SCRAM-SHA-256 --tls \ + -u docdbadmin -p "$DB_PASSWORD" --authenticationMechanism SCRAM-SHA-256 --tls \ --tlsAllowInvalidCertificates ``` @@ -141,9 +151,10 @@ Context: - Each order has: order_id, customer_id, status (pending/confirmed/shipped/ delivered/cancelled), payment_method, created_at, updated_at, shipping_city, total_amount. -- If you don't have a database connection, run commands through the shell: +- If you don't have a database connection, run commands through the shell + (the password is in my DB_PASSWORD environment variable): docker exec -u documentdb documentdb-local mongosh "localhost:10260/ecommerce" \ - -u docdbadmin -p Test1234 --authenticationMechanism SCRAM-SHA-256 --tls \ + -u docdbadmin -p "$DB_PASSWORD" --authenticationMechanism SCRAM-SHA-256 --tls \ --tlsAllowInvalidCertificates --quiet --eval '' Please run explain("executionStats"), tell me what the plan is doing, and @@ -192,7 +203,7 @@ db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 }) > > ```bash > docker exec -u documentdb documentdb-local mongosh "localhost:10260/ecommerce" \ -> -u docdbadmin -p Test1234 --authenticationMechanism SCRAM-SHA-256 --tls \ +> -u docdbadmin -p "$DB_PASSWORD" --authenticationMechanism SCRAM-SHA-256 --tls \ > --tlsAllowInvalidCertificates --quiet \ > --eval 'db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 })' > ``` @@ -235,7 +246,7 @@ The kit includes a ready-made test that runs several *before/after* comparisons for you and prints the results: ```bash -export DB_PASSWORD=Test1234 +# uses the same DB_PASSWORD you exported in Step 1 bash scenarios/ecommerce/query-perf-skill-test.sh ``` diff --git a/scenarios/contoso/README.md b/scenarios/contoso/README.md index b6c9360..284d381 100644 --- a/scenarios/contoso/README.md +++ b/scenarios/contoso/README.md @@ -54,7 +54,7 @@ into that container, so no host ports are required. ```bash # 0. the scripts require a password — export it once (or pass --password) -export DB_PASSWORD=Test1234 +export DB_PASSWORD='' # 1. seed the demo database (base size ~500 opportunities) bash scenarios/contoso/seed.sh # -> database "contoso" diff --git a/scenarios/contoso/seed.sh b/scenarios/contoso/seed.sh index ae4888b..89c93e8 100755 --- a/scenarios/contoso/seed.sh +++ b/scenarios/contoso/seed.sh @@ -34,7 +34,7 @@ while [[ $# -gt 0 ]]; do esac done -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } echo "Seeding base-size Contoso into '${DB}' (container: ${CONTAINER}) ..." docker cp "$DIR/contoso-seed.js" "${CONTAINER}:/tmp/contoso-seed.js" >/dev/null diff --git a/scenarios/ecommerce/README.md b/scenarios/ecommerce/README.md index 8deec6b..eb18a21 100644 --- a/scenarios/ecommerce/README.md +++ b/scenarios/ecommerce/README.md @@ -14,7 +14,7 @@ the diagnostic toolbox: `perf-advisor.sh`, `index-redundancy-finder.sh`, and ## Run ```bash -export DB_PASSWORD=Test1234 # or pass --password +export DB_PASSWORD='' # or pass --password bash scenarios/ecommerce/seed.sh # -> database "ecommerce" bash scripts/perf-advisor.sh --db ecommerce diff --git a/scenarios/ecommerce/query-perf-skill-test.sh b/scenarios/ecommerce/query-perf-skill-test.sh index 0969107..7397984 100755 --- a/scenarios/ecommerce/query-perf-skill-test.sh +++ b/scenarios/ecommerce/query-perf-skill-test.sh @@ -3,7 +3,7 @@ # against the seeded `ecommerce` dataset in a local DocumentDB container. # # Prereq: a running `documentdb-local` container with the ecommerce dataset seeded -# export DB_PASSWORD=Test1234 +# export DB_PASSWORD='' # bash scenarios/ecommerce/seed.sh # bash scenarios/ecommerce/query-perf-skill-test.sh # @@ -30,7 +30,7 @@ while [[ $# -gt 0 ]]; do esac done -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } [[ -f "$JS" ]] || { echo "Error: harness not found at $JS" >&2; exit 1; } # Copy the harness into the container and run it with the bundled mongosh. diff --git a/scenarios/ecommerce/seed.sh b/scenarios/ecommerce/seed.sh index 895f3cd..affbf46 100755 --- a/scenarios/ecommerce/seed.sh +++ b/scenarios/ecommerce/seed.sh @@ -2,7 +2,7 @@ # seed.sh — Generate the large "ecommerce" demo dataset for perf/diagnostic testing # Creates: customers (5K), products (2K), orders (50K), order_items (150K), # reviews (25K), inventory (4K) -# Usage: export DB_PASSWORD=Test1234; bash scenarios/ecommerce/seed.sh [--container NAME] [--password PASS] +# Usage: export DB_PASSWORD=''; bash scenarios/ecommerce/seed.sh [--container NAME] [--password PASS] set -uo pipefail CONTAINER_NAME="${CONTAINER_NAME:-documentdb-local}" @@ -22,7 +22,7 @@ while [[ $# -gt 0 ]]; do esac done -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } run_mongosh() { docker exec -u documentdb "$CONTAINER_NAME" mongosh \ diff --git a/scenarios/index-redundancy/README.md b/scenarios/index-redundancy/README.md index af2f9a3..047cd2b 100644 --- a/scenarios/index-redundancy/README.md +++ b/scenarios/index-redundancy/README.md @@ -15,7 +15,7 @@ findings to report. ## Run ```bash -export DB_PASSWORD=Test1234 # or pass --password +export DB_PASSWORD='' # or pass --password bash scenarios/index-redundancy/seed.sh # -> database "idx_test" bash scripts/index-redundancy-finder.sh --db idx_test bash scripts/index-redundancy-finder.sh --db idx_test --json # machine output diff --git a/scenarios/index-redundancy/seed.sh b/scenarios/index-redundancy/seed.sh index 03edd72..854ceba 100755 --- a/scenarios/index-redundancy/seed.sh +++ b/scenarios/index-redundancy/seed.sh @@ -3,7 +3,7 @@ # and unused indexes, so index-redundancy-finder.sh has findings to report. # # Usage: -# export DB_PASSWORD=Test1234 +# export DB_PASSWORD='' # bash scenarios/index-redundancy/seed.sh # -> database "idx_test" # bash scenarios/index-redundancy/seed.sh --db mydb --container NAME --password PASS set -uo pipefail @@ -26,7 +26,7 @@ while [[ $# -gt 0 ]]; do esac done -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } echo "Seeding redundant-index fixture into '${DB}' (container: ${CONTAINER}) ..." docker cp "$DIR/fixture-redundant-indexes.js" "${CONTAINER}:/tmp/fixture-redundant-indexes.js" >/dev/null diff --git a/scripts/data-integrity-check.sh b/scripts/data-integrity-check.sh index 50af17a..25af154 100755 --- a/scripts/data-integrity-check.sh +++ b/scripts/data-integrity-check.sh @@ -53,7 +53,7 @@ EOF done [[ -z "$DB" ]] && { echo "Error: --db is required"; exit 1; } -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } run_mongosh() { docker exec -u documentdb "$CONTAINER_NAME" mongosh \ diff --git a/scripts/document-bloat-advisor.sh b/scripts/document-bloat-advisor.sh index b0c59b9..1080f94 100755 --- a/scripts/document-bloat-advisor.sh +++ b/scripts/document-bloat-advisor.sh @@ -53,7 +53,7 @@ while [[ $# -gt 0 ]]; do esac done [[ -z "$DB" ]] && { echo "Error: --db is required" >&2; exit 1; } -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } run_mongosh() { docker exec "$CONTAINER_NAME" mongosh "localhost:${PORT}/${DB}" \ diff --git a/scripts/index-redundancy-finder.sh b/scripts/index-redundancy-finder.sh index 8b4a714..1f192b4 100755 --- a/scripts/index-redundancy-finder.sh +++ b/scripts/index-redundancy-finder.sh @@ -74,7 +74,7 @@ EOF done [[ -z "$DB" && "$ALL_DBS" != "true" ]] && { echo "Error: --db or --all-dbs is required"; exit 1; } -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } # ── Helpers ─────────────────────────────────────────────────────────── run_mongosh() { diff --git a/scripts/perf-advisor.sh b/scripts/perf-advisor.sh index 6bddea1..e2b4eae 100755 --- a/scripts/perf-advisor.sh +++ b/scripts/perf-advisor.sh @@ -71,7 +71,7 @@ EOF done [[ -z "$DB" && "$ALL_DBS" != "true" ]] && { echo "Error: --db or --all-dbs is required"; exit 1; } -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } # ── Helper functions ────────────────────────────────────────────────── run_mongosh() { diff --git a/scripts/toast-split-advisor.sh b/scripts/toast-split-advisor.sh index 46fc3bd..0cec7f4 100755 --- a/scripts/toast-split-advisor.sh +++ b/scripts/toast-split-advisor.sh @@ -67,7 +67,7 @@ while [[ $# -gt 0 ]]; do esac done [[ -z "$DB" ]] && { echo "Error: --db is required" >&2; exit 1; } -[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (local demo: export DB_PASSWORD=Test1234)." >&2; exit 1; } +[[ -z "$PASSWORD" ]] && { echo "Error: no password. Set DB_PASSWORD or pass --password (e.g. export DB_PASSWORD='')." >&2; exit 1; } run_mongosh() { docker exec "$CONTAINER_NAME" mongosh "localhost:${PORT}/${DB}" \ diff --git a/testing/conftest.py b/testing/conftest.py index 470a349..42dd47f 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -35,7 +35,7 @@ def require_container(container_name): DB_PASSWORD).""" if not kit.DB_PASSWORD: pytest.skip("No DB password configured — set DOCDB_PASSWORD or DB_PASSWORD " - "(local demo: export DB_PASSWORD=Test1234)") + "(e.g. export DB_PASSWORD='')") if not kit.container_running(container_name): pytest.skip(f"DocumentDB container '{container_name}' is not running " f"(start it, or pass --container / set DOCDB_CONTAINER)") From 2af0896c76862021e78c1bcbaa66aba03e8e6920 Mon Sep 17 00:00:00 2001 From: Lei Jiang Date: Mon, 10 Aug 2026 22:08:37 +0000 Subject: [PATCH 13/13] Fold the local-image caveat into the skill; drop the query-perf test report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The query-perf test write-up was a one-off validation report, not documentation: ~40% of it duplicated the quickstart (container setup, mongosh install, seeding, harness command, managed-Azure note) and it had a single inbound link, so its most valuable content was effectively unreachable. Move that content to where it is actually consumed — the explain reference the `documentdb-query-performance-tuning` skill loads: - On `documentdb-local`, index-backed sort and covered queries do NOT reproduce. An ideal ESR index still yields `SORT -> FETCH -> IXSCAN`, and a covering projection does not drop the `FETCH`. Enabling the experimental engine flags via ALTER SYSTEM (and rebuilding the index) does not change the Mongo-API plan, because the gateway pins them per session. Includes the table of gated flags and how to inspect them, plus the guidance to judge a local index by scan-stage documents/keys examined rather than by the SORT/FETCH disappearing. - The companion gotcha that executionTimeMillis is noisy on small result sets. Delete docs/query-perf-skill-test-instruction.md and repoint scenarios/ecommerce/README.md at the harness + quickstart. The reproducible artifact (query-perf-skill-test.js/.sh) is unchanged. docs/DIAGNOSTICS.md is kept: it documents the diagnostic-script toolbox, a different subsystem from the quickstart, and is linked from README/AGENTS/SKILLS. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 86722878-7b4c-4d57-933c-3d4188317703 --- ...f-skill-test-instruction-20260730180441.md | 323 ------------------ scenarios/ecommerce/README.md | 6 +- .../references/documentdb-explain-output.md | 47 +++ 3 files changed, 51 insertions(+), 325 deletions(-) delete mode 100644 docs/query-perf-skill-test-instruction-20260730180441.md diff --git a/docs/query-perf-skill-test-instruction-20260730180441.md b/docs/query-perf-skill-test-instruction-20260730180441.md deleted file mode 100644 index d142bd6..0000000 --- a/docs/query-perf-skill-test-instruction-20260730180441.md +++ /dev/null @@ -1,323 +0,0 @@ -# Query-Performance Skill Test — Instruction & Results - -> **Generated:** 2026-07-30 18:04:41 UTC · **Re-run (post-merge):** 2026-07-30 21:18 UTC — identical scan-work results; test 4.1 re-labeled to `documentdb-query-optimizer` after the `query-optimization` merge · **Status:** Public Preview (see repo `AGENTS.md`) -> **Article under test:** [Query Performance Tuning Guide](https://devblogs.microsoft.com/documentdb/query-performance-tuning-guide/) (Azure DocumentDB devblog) -> **Skills exercised:** `documentdb-query-performance-tuning`, `documentdb-query-optimizer` (two facets: index recommendation + explain-plan verification, the latter merged from the former `documentdb-query-optimization`), `documentdb-indexing` (`index-compound-esr`) - -This document is a **reproducible before/after test** that validates the query-performance -skills against a **real local DocumentDB container**. It takes a realistic slow query per case, -captures `explain("executionStats")` **before** any supporting index (the `COLLSCAN` baseline), -applies the index that skill recommends, and re-captures explain **after**. Every number below is -measured, not estimated. The 2026-07-30 21:18 UTC re-run (after `query-optimization` was merged -into `query-optimizer`) reproduced every scan-work figure exactly. - -**Reproducible test artifacts** (added with this doc): -- `scenarios/ecommerce/query-perf-skill-test.js` — the mongosh measurement harness -- `scenarios/ecommerce/query-perf-skill-test.sh` — the container runner - -> ### ⚠️ Key caveat (read first) -> The **headline optimization reproduces** here: every skill turns a 50,000-doc `COLLSCAN` -> into a selective `IXSCAN`. But two of the article's *further* optimizations — **index-backed -> sort** (removing the in-memory `SORT`) and **covered queries** (removing the `FETCH`) — **do -> not reproduce on the `documentdb-local` image**, because the experimental engine flags that -> enable them ship **off** and are **pinned per-session by the gateway** (so even `ALTER SYSTEM` -> can't flip them). These are managed **Azure DocumentDB** behaviors, per the article. Full, -> verified detail and the fuller list of gated flags is in [§6 Caveats](#6-honest-caveats-local-image-vs-managed-azure-documentdb). - ---- - -## 1. Is the `scenarios/` dataset a good fit? — Yes: `ecommerce` - -The existing **`ecommerce`** seed (`scenarios/ecommerce/seed.sh`) is an excellent match for the -article. Its `orders` collection is seeded with **50,000 documents and *no* secondary index** -(only the default `_id`), so the "before" state is a genuine full collection scan — exactly the -article's starting point. The document shape maps almost 1:1 onto the article's example: - -| Article field | `ecommerce.orders` field | Notes | -|---|---|---| -| `status: "shipped"` | `status` ∈ {pending, confirmed, shipped, delivered, cancelled} | `"shipped"` ≈ 10,127 / 50,000 docs (low cardinality) | -| `customerId: "C-4821"` | `customer_id` = `CUST_000000`…`CUST_004999` | 5,000 distinct → **high cardinality** (~10 orders each), like the article's customerId | -| `createdAt` | `created_at` (all in 2024) | supports the `$gte: ISODate("2024-01-01")` range + sort | -| (order total) | `total_amount` | used for the range-predicate (ESR) case | -| (city) | `shipping_city` ∈ 10 cities | used for the two-equality case | - -The only differences from the article are **field naming** (`customer_id`/`created_at` vs -`customerId`/`createdAt`) and **scale** (50K docs vs the article's 1M). The before/after story -is identical. - -> The other datasets are **not** a fit for this article: `contoso` targets large-text/TOAST -> bloat and `index-redundancy` targets duplicate/unused indexes — different diagnostics. - ---- - -## 2. Environment & prerequisites - -| Item | Value | -|---|---| -| Container image | `ghcr.io/microsoft/documentdb/documentdb-local@sha256:0fcf634531c1917ad0855ff9f4354aca0a5c5d8e435f08250568deb37eeb0ad5` | -| Container name | `documentdb-local` | -| Gateway (Mongo API) | container-internal `localhost:10260`, TLS, SCRAM-SHA-256 | -| PostgreSQL engine | container-internal `localhost:9712` | -| Credentials | user `docdbadmin`, from the `DB_PASSWORD` env var (choose your own) | -| mongosh client | `2.3.8` | - -> **Reproducibility gotcha:** this `documentdb-local` image **does not bundle a `mongosh` -> client**, but `scenarios/*/seed.sh` and the harness invoke `mongosh` via `docker exec`. Install -> it into the container once (root): -> ```bash -> docker exec -u root documentdb-local bash -lc ' -> cd /tmp && wget -q https://downloads.mongodb.com/compass/mongosh-2.3.8-linux-x64.tgz -O m.tgz && -> tar xzf m.tgz && cp mongosh-2.3.8-linux-x64/bin/mongosh /usr/local/bin/ && -> cp mongosh-2.3.8-linux-x64/bin/mongosh_crypt_v1.so /usr/local/lib/ 2>/dev/null; mongosh --version' -> ``` - -### Bring up + seed - -```bash -# 0. pick a password for the throwaway local container; everything below reads it -export DB_PASSWORD='' -# 1. start the container (host ports optional; seeders use docker exec) -docker run -d --name documentdb-local -e USERNAME=docdbadmin -e PASSWORD="$DB_PASSWORD" \ - ghcr.io/microsoft/documentdb/documentdb-local:latest -# 2. install mongosh into the container (see gotcha above) -# 3. seed the ecommerce dataset (~50K orders; takes ~1 min) -bash scenarios/ecommerce/seed.sh -``` - ---- - -## 3. How each skill is triggered (exact query words) - -All trigger phrases resolve **deterministically** in the router's skill space (Route B). - -> **Post-run update:** since this test was generated, the former `documentdb-query-optimization` -> rule skill was **merged into `documentdb-query-optimizer`** (its explain-verification content now -> lives in `query-optimizer/references/query-explain-plan.md`). So test 4.1 below is now the -> *explain-plan verification* facet of `query-optimizer`, and its trigger routes to `query-optimizer`. - -| Skill | Type | Exact trigger phrase | `kb-route.sh` result (verified) | -|---|---|---|---| -| `documentdb-query-optimizer` (explain-plan verification) | standalone | "use explain to check whether this query uses an index or does a collection scan" | → skill **`query-optimizer`** (7.10) ✅ | -| `documentdb-query-optimizer` | standalone | "optimize this query and recommend an index" | → skill **`query-optimizer`** (9.00) ✅ | -| `documentdb-query-performance-tuning` | standalone | "how do I read the explain output and apply the ESR rule to tune this query" | → skill **`query-performance-tuning`** (11.38) ✅ | -| `documentdb-indexing` (`index-compound-esr`) | rule-folder | "which compound index should I design for this filter, sort, and range query" | → skill **`indexing`** (7.50) ✅ | - -Try any phrase yourself: -```bash -bash knowledge-base/kb-route.sh --json "optimize this query and recommend an index" -``` - ---- - -## 4. Before/after results (measured) - -**How to read these.** Azure DocumentDB's planner is PostgreSQL-backed, so the **top-line** -`totalDocsExamined` is a *post-sort* count and can look tiny even during a full scan — the -article warns about exactly this. The honest "work" metric is **documents/keys examined at the -scan stage**, which the harness drills down to. `executionTimeMillis` is the **min of 5 runs** -but is noisy at this data size (see caveats). - -Each test runs against `ecommerce.orders` (50,000 docs). "Scan work" = docs examined by a -`COLLSCAN` vs keys examined by the `IXSCAN`. - -### 4.1 `documentdb-query-optimizer` (explain-plan verification, merged from `query-optimization`) - -**Goal:** explain() literacy — a `COLLSCAN` becomes an `IXSCAN` once an index supports the filter. - -```javascript -db.orders.find({ order_id: "ORD_012345" }) // single high-selectivity equality -db.orders.createIndex({ order_id: 1 }) // the fix -``` - -| Metric | Before | After | -|---|---|---| -| Scan stage | **COLLSCAN** | **IXSCAN** (`order_id_1`) → FETCH | -| Examined at scan | **50,000 docs** | **1 key** | -| Blocking SORT | – | – | -| nReturned | 1 | 1 | -| `executionTimeMillis` (min/5) | 9 ms | 0 ms | -| Top-line `totalDocsExamined` | 50,000 | 1 | - -**Scan work: 50,000 → 1 (≈50,000× fewer).** The cleanest case: no sort, so the plan collapses to -a bare index lookup + fetch. - -### 4.2 `documentdb-query-optimizer` — recommend a compound index - -**Goal:** recommend a compound index for a specific slow query (two equality fields + sort). - -```javascript -db.orders.find({ status: "shipped", shipping_city: "Seattle" }).sort({ created_at: -1 }) -db.orders.createIndex({ status: 1, shipping_city: 1, created_at: -1 }) // ESR: 2 equality, then sort -``` - -| Metric | Before | After | -|---|---|---| -| Scan stage | **COLLSCAN** | **IXSCAN** (`status_1_shipping_city_1_created_at_-1`) → FETCH | -| Examined at scan | **50,000 docs** | **998 keys** | -| Blocking SORT | yes | yes¹ | -| nReturned | 998 | 998 | -| `executionTimeMillis` (min/5) | 14 ms | 3 ms | - -**Scan work: 50,000 → 998 (≈50× fewer).** ¹The blocking `SORT` persists on this local image — see -caveat §6.1. - -### 4.3 `documentdb-query-performance-tuning` — the article's flagship query (ESR + covered) - -**Goal:** full ESR compound index with the **most-selective equality first**, plus the -covered-query variant. - -```javascript -db.orders.find({ - status: "shipped", - customer_id: "CUST_004087", - created_at: { $gte: ISODate("2024-01-01") } -}).sort({ created_at: -1 }) -db.orders.createIndex({ customer_id: 1, status: 1, created_at: -1 }) // customer_id first = most selective -// covered variant adds projection { _id:0, customer_id:1, status:1, created_at:1 } -``` - -| Metric | Before | After | After (covered) | -|---|---|---|---| -| Scan stage | **COLLSCAN** | **IXSCAN** → FETCH | **IXSCAN** → FETCH² | -| Examined at scan | **50,000 docs** | **10 keys** | **10 keys** | -| Blocking SORT | yes | yes¹ | yes¹ | -| nReturned | 10 | 10 | 10 | -| `executionTimeMillis` (min/5) | 12 ms | 28 ms³ | 28 ms³ | - -**Scan work: 50,000 → 10 (≈5,000× fewer).** Putting the high-cardinality `customer_id` first is -why this drops to 10 keys while §4.2 (leading with low-cardinality `status`) only drops to 998 — -a direct demonstration of the article's *"most selective field first"* rule. -²The covered projection does **not** remove the `FETCH` on this local image (caveat §6.1). -³Wall-clock is noisy on a 10-row result (caveat §6.2); the scan-work reduction is the real signal. - -### 4.4 `documentdb-indexing` — `index-compound-esr` (Range LAST) - -**Goal:** ESR with a range predicate — Equality, then Sort, then **Range last**. - -```javascript -db.orders.find({ status: "shipped", total_amount: { $gt: 500 } }).sort({ created_at: -1 }) -db.orders.createIndex({ status: 1, created_at: -1, total_amount: 1 }) // E → S → R -``` - -| Metric | Before | After | -|---|---|---| -| Scan stage | **COLLSCAN** | **IXSCAN** (`status_1_created_at_-1_total_amount_1`) → FETCH | -| Examined at scan | **50,000 docs** | **8,899 keys** | -| Blocking SORT | yes | yes¹ | -| nReturned | 8,899 | 8,899 | -| `executionTimeMillis` (min/5) | 24 ms | 42 ms³ | - -**Scan work: 50,000 → 8,899 (≈5.6× fewer).** The modest reduction is expected and instructive: -`status: "shipped"` + `total_amount > 500` is a **low-selectivity** predicate (most shipped -orders exceed \$500), so even a correct ESR index cannot narrow much. Selectivity, not index -correctness alone, bounds the win. - ---- - -## 5. Summary - -| Skill | Query shape | Recommended index | Scan work (before → after) | Reduction | -|---|---|---|---|---| -| `query-optimizer` (explain verification) | 1 equality, no sort | `{order_id:1}` | 50,000 → 1 | **≈50,000×** | -| `query-optimizer` | 2 equality + sort | `{status:1, shipping_city:1, created_at:-1}` | 50,000 → 998 | **≈50×** | -| `query-performance-tuning` | 2 equality + range + sort | `{customer_id:1, status:1, created_at:-1}` | 50,000 → 10 | **≈5,000×** | -| `indexing` | 1 equality + range + sort | `{status:1, created_at:-1, total_amount:1}` | 50,000 → 8,899 | **≈5.6×** | - -**Every skill turns a 50,000-document `COLLSCAN` into an `IXSCAN`.** The magnitude of the win -tracks **selectivity** exactly as the article predicts: high-cardinality leading equality fields -(`order_id`, `customer_id`) yield 1000×–50000× reductions; low-cardinality predicates -(`status`, coarse ranges) yield single- to double-digit reductions. - ---- - -## 6. Honest caveats (local image vs managed Azure DocumentDB) - -### 6.1 Index-backed sort and covered queries are gated OFF on `documentdb-local` - -The article's *further* optimizations — **eliminating the in-memory `SORT`** (index-backed sort) -and **eliminating the `FETCH`** (covered query) — **did not reproduce** on this local image. In -every "after" plan with a sort, a blocking `SORT` stage remained, and the covered-query variant -(§4.3) kept its `FETCH`. This is a **local-image configuration limitation, not a skill error.** - -**What I verified (measured):** -1. With the defaults, the flagship "after" plan is `SORT → FETCH → IXSCAN` and the covered - variant is byte-for-byte identical to the non-covered one (`FETCH` never drops). -2. I enabled the full experimental combo at the engine level — - `enableNewCompositeIndexOpClass=on`, `defaultUseCompositeOpClass=on`, - `enableIndexOrderbyPushdown=on`, `forceRumIndexScantoBitmapHeapScan=off` — via - `ALTER SYSTEM … ; SELECT pg_reload_conf();`, confirmed each shows `on` in a **fresh psql - session**, then **dropped and recreated** the index so it would be built under the new op - class. The Mongo-API `explain` **still** returned `SORT → FETCH → IXSCAN`. -3. Conclusion: **the gateway pins these settings per-session** (see its startup - *"Dynamic configurations loaded"* log), so PostgreSQL-level `ALTER SYSTEM` does not change the - Mongo-API query path. On the local image these experimental flags are simply off. (I reset all - four GUCs back to their defaults afterward; the harness itself never touches engine config.) - -**The relevant flags gated OFF on `documentdb-local`** (names/descriptions are the engine's own — -several literally say *"new experimental"*). The article demonstrates the corresponding behavior on -managed **Azure DocumentDB**, implying they are enabled there. - -| Engine flag | Local default | What it governs | Relevance to the article | Evidence | -|---|---|---|---|---| -| `documentdb.enableIndexOrderbyPushdown` | `off` | pushes `ORDER BY` into the composite index | **index-backed sort** (removes the blocking `SORT`) | ✅ measured: `SORT` persists | -| `documentdb.enableNewCompositeIndexOpClass` | `off` | the "new experimental composite index opclass" | prerequisite for ordered **and** covering composite indexes | ✅ measured: combo-on + index rebuild still `SORT`/`FETCH` | -| `documentdb.defaultUseCompositeOpClass` | `off` | whether default `createIndex` builds use that op class | so the recommended index is ordered/covering | ⬤ config-observed | -| `documentdb.enableSortbyIdPushDownToPrimaryKey` | `off` | pushes `sort({_id})` onto the primary key | index-backed sort on `_id` | ⬤ config-observed | -| `documentdb.forceRumIndexScantoBitmapHeapScan` | `on` | forces a bitmap heap scan (always re-fetches heap tuples) | **prevents index-only / covered scans** → `FETCH` always present | ✅ measured: covered variant keeps `FETCH` | -| `documentdb.enableNewSelectivityMode` | `off` | the "new selectivity logic" in the planner | the article's explain surfaces `indexCosts` / `selectivity` / `correlation` | ⬤ config-observed | -| `documentdb.enableMultiIndexRumJoin` | `off` | intersecting multiple indexes for one query | combining indexes vs one compound index | ⬤ config-observed | -| `documentdb.enablePrimaryKeyCursorScan` | `off` | primary-key cursor scan for streaming cursors | faster `_id`-ordered cursor scans | ⬤ config-observed | - -✅ = behavior measured in this test · ⬤ = observed in engine config (`pg_settings`), not separately behavior-tested here. - -**Beyond query tuning**, the same image also ships these adjacent performance features off (not -exercised by this test, listed for completeness): `documentdb.enable_force_push_vector_index=off` -and `documentdb.enableVectorPreFilterV2=off` (vector-search push-down / pre-filter v2). HNSW, -product-quantization and half-precision vector compression **are** on. - -> **How to see the full set yourself:** -> ```bash -> docker exec documentdb-local psql -h localhost -p 9712 -U documentdb -d postgres \ -> -c "SELECT name, setting, short_desc FROM pg_settings WHERE name LIKE 'documentdb.%' ORDER BY name;" -> ``` - -### 6.2 `executionTimeMillis` is noisy at this scale - -On small result sets some "after" timings exceed "before" (e.g. §4.3, 12 ms → 28 ms) due to plan -caching, index-page warmup, and sub-millisecond measurement noise. This is itself one of the -article's lessons — *time can look fine while the query does far too much work* — so this test -reports **documents/keys examined at the scan stage** as the stable, honest signal. - -### 6.3 Scale - -50,000 docs vs the article's 1,000,000. The plan **shapes** and scan-work **ratios** are what -matter and are scale-independent; absolute milliseconds are not comparable. - ---- - -## 7. Reproduce this test - -```bash -# prerequisites: container up + mongosh installed + ecommerce seeded (see §2) -export DB_PASSWORD='' -bash scenarios/ecommerce/query-perf-skill-test.sh # prints the human report + a JSON block -``` - -The harness (`scenarios/ecommerce/query-perf-skill-test.js`) is self-contained: for each skill it -drops every non-`_id` index on `orders` (forcing the baseline), measures before, creates the -recommended index, measures after (+ the covered variant for the flagship), and drops the index — -so it is **idempotent** and leaves `orders` with only its default `_id` index. - ---- - -## 8. Verdict - -- **Dataset fit:** `ecommerce.orders` is a strong, faithful stand-in for the article's `orders` - example — no new dataset needed. -- **Skills validated:** all four test cases (across three skills — `query-optimizer` covers two - facets) turn a `COLLSCAN` into an `IXSCAN`, with the reduction scaling by selectivity exactly as - the ESR guidance predicts. The recommended indexes and trigger phrases are the ones the skills - ship. -- **Faithful limitation:** index-backed-sort and covered-query removal are environment-gated on - the local image; they are Azure-DocumentDB-managed behaviors (per the article), not something - the local test can demonstrate. diff --git a/scenarios/ecommerce/README.md b/scenarios/ecommerce/README.md index eb18a21..71633e6 100644 --- a/scenarios/ecommerce/README.md +++ b/scenarios/ecommerce/README.md @@ -26,8 +26,10 @@ bash scenarios/ecommerce/query-perf-skill-test.sh ``` The query-perf test harness (`query-perf-skill-test.js` / `.sh`) validates the -query-performance skills against this dataset; see the generated instruction doc -under [`docs/`](../../docs/) (`query-perf-skill-test-instruction-*.md`). +query-performance skills against this dataset: for each case it forces a +`COLLSCAN` baseline, creates the index that skill recommends, re-measures, and +drops it again — so it is idempotent. For a guided walkthrough of the same +before/after, see [`docs/quickstart-find-and-fix-slow-queries.md`](../../docs/quickstart-find-and-fix-slow-queries.md). Prereq: a running `documentdb-local` container — see the repo [`README.md`](../../README.md#quickstart) *Quickstart*. Overrides: `--container`, diff --git a/skills/query-performance-tuning/references/documentdb-explain-output.md b/skills/query-performance-tuning/references/documentdb-explain-output.md index b72b760..55e4e9c 100644 --- a/skills/query-performance-tuning/references/documentdb-explain-output.md +++ b/skills/query-performance-tuning/references/documentdb-explain-output.md @@ -120,6 +120,53 @@ leaving a bare `IXSCAN`. | `SORT` stage present | absent | present (blocking in-memory sort) | | `FETCH` stage present | absent = covered query | present (unavoidable if you need the full doc) | +## ⚠️ On `documentdb-local`, index-backed sort and covered queries do NOT reproduce + +If you are testing against the **local** container image +(`ghcr.io/microsoft/documentdb/documentdb-local`) rather than managed Azure +DocumentDB, expect a **residual `SORT` and `FETCH` even with a perfect ESR +index**. Do not diagnose this as a bad index — it is a configuration difference +in the local image. + +Measured on the local image: with an ideal ESR index the plan stays +`SORT → FETCH → IXSCAN`, and adding a covering projection does **not** remove the +`FETCH`. Enabling the experimental engine flags below via +`ALTER SYSTEM … ; SELECT pg_reload_conf();` — and even dropping and recreating the +index so it builds under the new op class — **does not change the Mongo-API plan**: +the gateway pins these settings per session (see its startup *"Dynamic +configurations loaded"* log). + +| Engine flag | Local default | What it governs | +|---|---|---| +| `documentdb.enableIndexOrderbyPushdown` | `off` | pushes `ORDER BY` into the composite index → **index-backed sort** | +| `documentdb.enableNewCompositeIndexOpClass` | `off` | the "new experimental composite index opclass" — prerequisite for ordered **and** covering composite indexes | +| `documentdb.defaultUseCompositeOpClass` | `off` | whether default `createIndex` builds use that op class | +| `documentdb.enableSortbyIdPushDownToPrimaryKey` | `off` | pushes `sort({_id})` onto the primary key | +| `documentdb.forceRumIndexScantoBitmapHeapScan` | `on` | forces a bitmap heap scan (always re-fetches heap tuples) → **prevents index-only / covered scans** | +| `documentdb.enableNewSelectivityMode` | `off` | newer planner selectivity logic (`indexCosts` / `selectivity` / `correlation`) | +| `documentdb.enableMultiIndexRumJoin` | `off` | intersecting multiple indexes for one query | +| `documentdb.enablePrimaryKeyCursorScan` | `off` | primary-key cursor scan for streaming cursors | + +Inspect the full set yourself: + +```bash +docker exec documentdb-local psql -h localhost -p 9712 -U documentdb -d postgres \ + -c "SELECT name, setting, short_desc FROM pg_settings WHERE name LIKE 'documentdb.%' ORDER BY name;" +``` + +**What this means for tuning locally:** the primary lever — replacing a `COLLSCAN` +with a selective `IXSCAN` — reproduces fully, so judge a local index by +**documents/keys examined at the scan stage**, not by whether the `SORT`/`FETCH` +disappeared. On managed Azure DocumentDB these paths are enabled, which is why the +published guidance shows the `SORT`/`FETCH` vanishing. + +### Related gotcha: `executionTimeMillis` is noisy on small result sets + +On small matches an "after" timing can exceed the "before" (plan caching, +index-page warmup, sub-millisecond noise). That is itself the core lesson of this +skill — *time can look fine while the query does far too much work* — so judge +improvements by **scan-stage documents/keys examined**, which is stable. + ## References - [How to read explain output](https://learn.microsoft.com/en-us/azure/documentdb/how-to-read-explain-output)