From 883172043009cf7c5bf6d5e6f8eb769b3b611163 Mon Sep 17 00:00:00 2001 From: I543501 Date: Thu, 30 Jul 2026 12:57:31 +0200 Subject: [PATCH 01/49] init --- .gitignore | 4 + evals/config.json | 35 ++++ evals/data/baseline.json | 313 +++++++++++++++++++++++++++++++ evals/data/golden-set.json | 85 +++++++++ evals/docs/METRICS.md | 87 +++++++++ evals/docs/README.md | 169 +++++++++++++++++ evals/lib/cli.js | 108 +++++++++++ evals/lib/config.js | 124 ++++++++++++ evals/lib/ids.js | 63 +++++++ evals/lib/metrics.js | 112 +++++++++++ evals/lib/retriever.js | 46 +++++ evals/lib/runner.js | 229 ++++++++++++++++++++++ evals/run.js | 11 ++ evals/runs/.gitkeep | 0 evals/tests/unit/config.test.js | 101 ++++++++++ evals/tests/unit/metrics.test.js | 109 +++++++++++ evals/tests/unit/runner.test.js | 187 ++++++++++++++++++ package.json | 5 +- 18 files changed, 1787 insertions(+), 1 deletion(-) create mode 100644 evals/config.json create mode 100644 evals/data/baseline.json create mode 100644 evals/data/golden-set.json create mode 100644 evals/docs/METRICS.md create mode 100644 evals/docs/README.md create mode 100644 evals/lib/cli.js create mode 100644 evals/lib/config.js create mode 100644 evals/lib/ids.js create mode 100644 evals/lib/metrics.js create mode 100644 evals/lib/retriever.js create mode 100644 evals/lib/runner.js create mode 100644 evals/run.js create mode 100644 evals/runs/.gitkeep create mode 100644 evals/tests/unit/config.test.js create mode 100644 evals/tests/unit/metrics.test.js create mode 100644 evals/tests/unit/runner.test.js diff --git a/.gitignore b/.gitignore index 99abc08..e4f4e16 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ models/ embeddings/ node_modules/ .claude + +# Transient eval run reports (data/baseline.json is committed; individual runs are not) +evals/runs/* +!evals/runs/.gitkeep diff --git a/evals/config.json b/evals/config.json new file mode 100644 index 0000000..133d7f2 --- /dev/null +++ b/evals/config.json @@ -0,0 +1,35 @@ +{ + "$comment": "Single source of configuration for the CAP MCP retrieval evals. Every value here is overridable via env vars (EVAL_*) — see lib/config.js. Paths are relative to the evals/ directory. NOTE: metrics are computed at 'k' and the baseline was captured at a specific 'k'; if you change 'k', re-capture the baseline (npm run evals:baseline) or deltas will be misleading.", + + "k": 5, + + "paths": { + "goldenSet": "data/golden-set.json", + "baseline": "data/baseline.json", + "runsDir": "runs" + }, + + "corpus": { + "corpus_version": "capire@unknown", + "index_rev": 0, + "embedding_model": "Xenova/all-MiniLM-L6-v2" + }, + + "offline": true, + + "gates": { + "$comment": "Gated metrics fail the run (non-zero exit) when value < threshold. null = reported only, never fails. Thresholds are derived empirically from a baseline run — not hardcoded standards.", + "recall_at_k": 0.8, + "mrr": 0.5, + "hit_rate_at_k": 0.8, + "precision_at_k": null, + "ndcg_at_k": null + }, + + "output": { + "$comment": "Run-output hygiene. Each run writes runs/eval-run-.json plus updates runs/latest.json. Old timestamped runs are pruned to keepRuns (0 = keep none, only latest.json; -1 = keep all). writeTimestamped=false writes only latest.json.", + "writeTimestamped": true, + "keepRuns": 20, + "latestName": "latest.json" + } +} diff --git a/evals/data/baseline.json b/evals/data/baseline.json new file mode 100644 index 0000000..8f941e7 --- /dev/null +++ b/evals/data/baseline.json @@ -0,0 +1,313 @@ +{ + "run_id": "2026-07-30T07:11:55Z_2efc13", + "deterministic": true, + "config": { + "corpus_version": "capire@2026-07-28", + "index_rev": 4471, + "embedding_model": "Xenova/all-MiniLM-L6-v2", + "golden_set": "cap-golden-v1", + "golden_set_size": 10, + "k": 5 + }, + "baseline_run_id": null, + "aggregate": { + "recall_at_k": { + "value": 1, + "baseline": null, + "delta": null, + "gate": 0.8, + "status": "pass" + }, + "mrr": { + "value": 0.78, + "baseline": null, + "delta": null, + "gate": 0.5, + "status": "pass" + }, + "precision_at_k": { + "value": 0.38, + "baseline": null, + "delta": null, + "gate": null, + "status": "info" + }, + "hit_rate_at_k": { + "value": 1, + "baseline": null, + "delta": null, + "gate": 0.8, + "status": "pass" + }, + "ndcg_at_k": { + "value": 0.84, + "baseline": null, + "delta": null, + "gate": null, + "status": "info" + } + }, + "overall_status": "pass", + "gated_failures": [], + "diagnosis": "no_regression", + "per_question": [ + { + "id": "cap-001", + "question": "How do I define a composition of one entity in CDS?", + "relevant_doc_ids": [ + "cds-language-compiler-managed-compositions-for#45c7585d", + "march-2025-cds-language-compiler-actions#e9b209b0" + ], + "retrieved_ids": [ + "cds-language-compiler-managed-compositions-for#45c7585d", + "february-2025-cds-language-compiler-type#59541813", + "march-2025-cds-language-compiler-actions#e9b209b0", + "september-2020-cds-language-compiler-simplified#ab9bdaef", + "working-with-cds-models-examples#80d4f7b9" + ], + "relevant_hits_at_rank": [ + 1, + 3 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.4, + "mrr": 1, + "hit_rate_at_k": 1, + "ndcg_at_k": 0.92 + } + }, + { + "id": "cap-002", + "question": "How do I expose a service via OData v4?", + "relevant_doc_ids": [ + "application-services-handling-crud-events-odata#be5c651d" + ], + "retrieved_ids": [ + "application-services-handling-crud-events-odata#be5c651d", + "april-2023-java-open-types-in#336e155d", + "remote-services-remote-odata-services#aa8f277b", + "remote-services-remote-odata-services-configuring#02a0f5af", + "remote-services-remote-odata-services-using#3d355412" + ], + "relevant_hits_at_rank": [ + 1 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.2, + "mrr": 1, + "hit_rate_at_k": 1, + "ndcg_at_k": 1 + } + }, + { + "id": "cap-003", + "question": "What is the difference between managed and unmanaged compositions?", + "relevant_doc_ids": [ + "cds-language-compiler-managed-compositions-for#45c7585d" + ], + "retrieved_ids": [ + "cds-language-compiler-managed-compositions-for#45c7585d", + "march-2025-cds-language-compiler-actions#e9b209b0", + "building-applications-code-generation-for-typed#4d07c685", + "change-tracking-identifiers-for-compositions#3eed4a12", + "july-2021-command-line-toolkit-importer#95a87d5f" + ], + "relevant_hits_at_rank": [ + 1 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.2, + "mrr": 1, + "hit_rate_at_k": 1, + "ndcg_at_k": 1 + } + }, + { + "id": "cap-004", + "question": "How do I add custom error handling in a service handler?", + "relevant_doc_ids": [ + "error-handler-the-following-example-of#c12c0e50", + "indicating-errors-exceptions#668e3c47", + "indicating-errors-messages#199f60ee" + ], + "retrieved_ids": [ + "march-2024-capire-documentation-java-wrapping#4d9c6ada", + "error-handler-the-following-example-of#c12c0e50", + "java-sdk-error-response-handler#10823207", + "indicating-errors-exceptions#668e3c47", + "indicating-errors-messages#199f60ee" + ], + "relevant_hits_at_rank": [ + 2, + 4, + 5 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.6, + "mrr": 0.5, + "hit_rate_at_k": 1, + "ndcg_at_k": 0.68 + } + }, + { + "id": "cap-005", + "question": "How do I define an association in CDL?", + "relevant_doc_ids": [ + "conceptual-definition-language-cdl-language-preliminaries#f6b0b479", + "june-2022-cds-language-compiler-define#29dacdf9" + ], + "retrieved_ids": [ + "june-2023-cds-language-type-of#ed11244e", + "june-2022-cds-language-compiler-define#29dacdf9", + "september-2023-cds-language-compiler-publish#fb87e5b4", + "conceptual-definition-language-cdl-language-preliminaries#f6b0b479", + "february-2021-cds-language-compiler-resolving#bcaa2618" + ], + "relevant_hits_at_rank": [ + 2, + 4 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.4, + "mrr": 0.5, + "hit_rate_at_k": 1, + "ndcg_at_k": 0.651 + } + }, + { + "id": "cap-006", + "question": "How do I add authorization with @requires and @restrict?", + "relevant_doc_ids": [ + "cap-level-authorization-role-based-access#8013298a", + "cap-level-authorization-instance-based-access#d17c0bb7", + "authentication-authorization-enforcement#dceddd27" + ], + "retrieved_ids": [ + "september-2024-node-js-instance-based#67280efe", + "cap-level-authorization-best-practices-separation#4b03a8a6", + "cap-level-authorization-role-based-access#8013298a", + "cap-level-authorization-instance-based-access#d17c0bb7", + "authentication-authorization-enforcement#dceddd27" + ], + "relevant_hits_at_rank": [ + 3, + 4, + 5 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.6, + "mrr": 0.333, + "hit_rate_at_k": 1, + "ndcg_at_k": 0.618 + } + }, + { + "id": "cap-007", + "question": "How do I run a CAP project locally with cds watch?", + "relevant_doc_ids": [ + "cds-command-line-interface-cli#ca38469f", + "cds-editors-and-ides-visual-studio#8e7f9c67" + ], + "retrieved_ids": [ + "cds-editors-and-ides-visual-studio#8e7f9c67", + "cds-command-line-interface-cli#ca38469f", + "introduction-to-cap-jumpstart-grow-as#8828cf14", + "cap-plugins-enhancements-as-cds-plugins#9b8f364d", + "tools-cds-watch-with-include-and#dddcb009" + ], + "relevant_hits_at_rank": [ + 1, + 2 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.4, + "mrr": 1, + "hit_rate_at_k": 1, + "ndcg_at_k": 1 + } + }, + { + "id": "cap-008", + "question": "How do I connect to a SQLite database in CAP?", + "relevant_doc_ids": [ + "june-2024-node-js-new-database#fece26f4", + "june-2023-node-js-new-database#28841c9b" + ], + "retrieved_ids": [ + "june-2023-node-js-plugins#55b57b1d", + "june-2023-node-js-new-database#28841c9b", + "june-2024-node-js-new-database#fece26f4", + "migration-guides-cap-java-classic-to#e4629f17", + "deploy-multitenant-saas-applications-test-drive#c56aa4da" + ], + "relevant_hits_at_rank": [ + 2, + 3 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.4, + "mrr": 0.5, + "hit_rate_at_k": 1, + "ndcg_at_k": 0.693 + } + }, + { + "id": "cap-009", + "question": "How do I define a service with entities projected from the data model?", + "relevant_doc_ids": [ + "core-concepts-of-cap-services-services#0c60ad11" + ], + "retrieved_ids": [ + "core-concepts-of-cap-services-services#0c60ad11", + "appendices-about-extension-models-extending-the#e9efd453", + "configuring-required-services-cds-requires-srv#b08a370b", + "march-2023-calculated-elements#414585f2", + "temporal-data-serving-temporal-data#e056fd32" + ], + "relevant_hits_at_rank": [ + 1 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.2, + "mrr": 1, + "hit_rate_at_k": 1, + "ndcg_at_k": 1 + } + }, + { + "id": "cap-010", + "question": "How do I add localized texts to entities in CDS?", + "relevant_doc_ids": [ + "localized-data-declaring-localized-data#e94844d3", + "cds-language-compiler-extending-the-generated#addfdfcb" + ], + "retrieved_ids": [ + "localized-data-declaring-localized-data#e94844d3", + "localized-data-behind-the-scenes#a88dbf68", + "localized-data-behind-the-scenes#85ea28a8", + "cds-language-compiler-reuse-type-sap#9b707792", + "cds-language-compiler-extending-the-generated#addfdfcb" + ], + "relevant_hits_at_rank": [ + 1, + 5 + ], + "metrics": { + "recall_at_k": 1, + "precision_at_k": 0.4, + "mrr": 1, + "hit_rate_at_k": 1, + "ndcg_at_k": 0.85 + } + } + ] +} diff --git a/evals/data/golden-set.json b/evals/data/golden-set.json new file mode 100644 index 0000000..5d00fe7 --- /dev/null +++ b/evals/data/golden-set.json @@ -0,0 +1,85 @@ +{ + "golden_set": "cap-golden-v1", + "description": "Frozen golden set for CAP MCP search_docs retrieval eval. Relevance ('relevant_doc_ids') is authored ONCE by a human against the capire@2026-07 index and stored here; it is never recomputed at eval time. IDs are stable content-derived identifiers (see evals/ids.js and METRICS.md), not chunk array indices. If a corpus re-index changes a labelled chunk's text, its id changes and the runner's pre-flight check will flag the stale id — refresh per evals/README.md.", + "questions": [ + { + "id": "cap-001", + "question": "How do I define a composition of one entity in CDS?", + "relevant_doc_ids": [ + "cds-language-compiler-managed-compositions-for#45c7585d", + "march-2025-cds-language-compiler-actions#e9b209b0" + ] + }, + { + "id": "cap-002", + "question": "How do I expose a service via OData v4?", + "relevant_doc_ids": [ + "application-services-handling-crud-events-odata#be5c651d" + ] + }, + { + "id": "cap-003", + "question": "What is the difference between managed and unmanaged compositions?", + "relevant_doc_ids": [ + "cds-language-compiler-managed-compositions-for#45c7585d" + ] + }, + { + "id": "cap-004", + "question": "How do I add custom error handling in a service handler?", + "relevant_doc_ids": [ + "error-handler-the-following-example-of#c12c0e50", + "indicating-errors-exceptions#668e3c47", + "indicating-errors-messages#199f60ee" + ] + }, + { + "id": "cap-005", + "question": "How do I define an association in CDL?", + "relevant_doc_ids": [ + "conceptual-definition-language-cdl-language-preliminaries#f6b0b479", + "june-2022-cds-language-compiler-define#29dacdf9" + ] + }, + { + "id": "cap-006", + "question": "How do I add authorization with @requires and @restrict?", + "relevant_doc_ids": [ + "cap-level-authorization-role-based-access#8013298a", + "cap-level-authorization-instance-based-access#d17c0bb7", + "authentication-authorization-enforcement#dceddd27" + ] + }, + { + "id": "cap-007", + "question": "How do I run a CAP project locally with cds watch?", + "relevant_doc_ids": [ + "cds-command-line-interface-cli#ca38469f", + "cds-editors-and-ides-visual-studio#8e7f9c67" + ] + }, + { + "id": "cap-008", + "question": "How do I connect to a SQLite database in CAP?", + "relevant_doc_ids": [ + "june-2024-node-js-new-database#fece26f4", + "june-2023-node-js-new-database#28841c9b" + ] + }, + { + "id": "cap-009", + "question": "How do I define a service with entities projected from the data model?", + "relevant_doc_ids": [ + "core-concepts-of-cap-services-services#0c60ad11" + ] + }, + { + "id": "cap-010", + "question": "How do I add localized texts to entities in CDS?", + "relevant_doc_ids": [ + "localized-data-declaring-localized-data#e94844d3", + "cds-language-compiler-extending-the-generated#addfdfcb" + ] + } + ] +} diff --git a/evals/docs/METRICS.md b/evals/docs/METRICS.md new file mode 100644 index 0000000..0a47856 --- /dev/null +++ b/evals/docs/METRICS.md @@ -0,0 +1,87 @@ +# CAP MCP RAG — Eval Metrics + +Deterministic, code-only retrieval evaluation for the CAP MCP server's `search_docs` +RAG pipeline. **No LLM is used anywhere** — in generation or in scoring. Every metric +is pure arithmetic computed from a frozen golden set whose relevance labels are +authored once by a human and stored in `relevant_doc_ids`. + +## Why these metrics (issue TODO #1 — alternatives to pass@k) + +`pass@k` in the sense of "did an agent complete the task in k attempts" requires a +correctness oracle. With **no LLM judge**, the only honest, deterministic collapse of +that idea is **Hit-Rate@K** — "did at least one relevant doc appear in the top-K?". +So there is deliberately **no separate `pass@k` concept and no LLM call**. Instead we +compute the standard information-retrieval metrics, which also let us attribute a +regression to a specific stage of the pipeline: + +| Metric | Formula (binary relevance) | What a drop tells you | +|---|---|---| +| **Recall@K** | `|relevant ∩ top-K| / |relevant|` | The right chunk isn't being retrieved at all → **chunking / embedding** regression | +| **Precision@K** | `|relevant ∩ top-K| / K` | Top-K is padded with irrelevant docs → **noise / padding** | +| **MRR** | `1 / rank(first relevant)`, `0` if none in top-K | Relevant chunk retrieved but ranked lower → **ranking / scoring** regression | +| **Hit-Rate@K** | `1` if `≥1` relevant in top-K else `0` | Cheapest smoke signal; subsumes "pass@k" without a judge | +| **nDCG@K** | `DCG@K / IDCG@K`, binary gains | Ordering degraded even if the set is right → **ranking / scoring** | + +`DCG@K = Σ_{i=1..K} rel_i / log2(i + 1)` (rank `i` is 1-based; `rel_i ∈ {0,1}`). +`IDCG@K` is `DCG@K` of the ideal ordering (all relevant docs first). `nDCG@K = 0` +when the question has no relevant docs in the index (should never happen — see the +pre-flight check). + +Each metric is computed **per question** and then **averaged (arithmetic mean)** over +the golden set to produce the aggregate. + +### Diagnosis (code-derived, first match wins) + +From the aggregate deltas vs. the baseline: + +1. Recall down → `recall_down → chunking/embedding regression` +2. Recall stable/up but MRR or nDCG down → `recall_stable_mrr_down → ranking/scoring regression` +3. Recall & MRR ok but Precision down → `precision_down → top-K padded with noise` +4. Nothing regressed → `no_regression` + +## Gated vs. reported metrics + +- **Gated** (a drop below threshold fails the run, non-zero exit): `recall_at_k`, `mrr`, `hit_rate_at_k`. +- **Reported only** (`gate: null`, never fails the run): `precision_at_k`, `ndcg_at_k`. + +Thresholds live in `config.json` (`gates`) and are **derived empirically from the first baseline +run** — they are not hardcoded "industry standard" numbers. Promote a known-good +`eval-run-.json` to `baseline.json`, then set each gate at/below the baseline value +with a small margin. + +## Stable identifiers (why not chunk index) + +The corpus re-indexes and chunk array positions shift, so ground truth must NOT be +keyed on the array index. The chunk store has no built-in id, so we derive a stable id +from the chunk **content**: + +``` +docId = "