Fold the eval-source workspace into evals (M2)

.github/workflows/workspace-selftest.yml

@@ -0,0 +1,31 @@
+name: workspace-selftest
+
+# Free self-tests of the eval-source workspace glue (workspace/README.md):
+# shell + node stdlib only — no API keys, no model spend, no docker, no
+# pnpm install. Submodules are fetched so the hooks/ab tests exercise the
+# real repos instead of skipping.
+on:
+  pull_request:
+    paths:
+      - "workspace/**"
+      - "mise.toml"
+      - ".github/workflows/workspace-selftest.yml"
+
+jobs:
+  selftest:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
+        with:
+          submodules: recursive
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .node-version
+      - name: status self-test
+        run: bash workspace/scripts/status.test.sh
+      - name: hooks self-test
+        run: bash workspace/scripts/hooks.test.sh
+      - name: ab self-test
+        run: bash workspace/scripts/ab.test.sh

.gitignore

@@ -6,3 +6,8 @@ dist/
 results/*/
 .sync-tmp/
 
+# eval-source workspace glue (workspace/README.md)
+/supabase/
+/results-ab/
+/.docs-index-stamp.json
+/.publish/

.gitmodules

@@ -4,3 +4,4 @@
 [submodule "submodules/mcp"]
 	path = submodules/mcp
 	url = git@github.com:supabase/mcp.git
+	ignore = all

README.md

@@ -177,6 +177,16 @@ Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/co
 - **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network) under `.claude/skills/`. When a task matches, the agent reads `.claude/skills/<name>/SKILL.md` (and any files it references) with its file tools.
 - **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name.
 
+## Eval-source workspace (docs / skills / MCP loops)
+
+The glue for testing changes to the agent's *inputs* — docs pages, skills,
+and MCP server source — lives in [`workspace/`](workspace/README.md): edit a
+source, run the affected evals against the local change, or run a head-to-head
+A/B (`mise run ab`) with per-arm provenance receipts. Tasks are driven by
+`mise` from the repo root (`mise run status` is the bootstrap probe). The
+docs monorepo stays an opt-in sparse clone; the MCP server is the pinned
+`submodules/mcp` (see "Running against an exact MCP server revision" above).
+
 ## Framework Checks
 
 ```bash

demo/canary-eval/EVAL.ts

@@ -0,0 +1,19 @@
+import { type CheckResult, type ToolScorer } from '@supabase-evals/core';
+
+// eval-workspace docs-discriminator demo: '@supabase/pinniped' is a fictional package
+// planted ONLY in the local docs by scripts/ab-demo.sh. It cannot come from model
+// priors or CLI scaffolding, so naming it proves the agent retrieved the answer
+// from the local docs index — and removing it (the A/B baseline) proves the doc
+// was the cause.
+const scorer: ToolScorer = async (ctx) => {
+  const report = ctx.agentReport ?? '';
+  const checks: CheckResult[] = [
+    {
+      name: 'named the docs-only package @supabase/pinniped',
+      passed: /pinniped/i.test(report),
+    },
+  ];
+  return { passed: checks[0].passed, checks };
+};
+
+export default scorer;

demo/canary-eval/PROMPT.md

@@ -0,0 +1,13 @@
+---
+stage: investigate
+suite: other
+interface: mcp
+product:
+  - edge-functions
+topic:
+  - sdk
+---
+
+I'm building a Supabase-managed cron/queue worker on the internal "Nimbus"
+runtime. Search the Supabase documentation and tell me the exact JavaScript
+helper package it recommends for this case. Name the package precisely.

mise.toml

@@ -0,0 +1,107 @@
+# supabase/evals — mise tasks (Supabase uses mise throughout; never a justfile).
+# The eval-source workspace glue lives in workspace/ (see workspace/README.md):
+# it wires docs, skills, and MCP-server sources into this harness for
+# edit -> eval loops. Tasks run from the repo root. Flag-style args need `--`:
+#   mise run eval -- --eval <id> --experiment <exp>
+
+[tools]
+node = "22"
+pnpm = "10.24.0"
+
+[tasks.setup]
+description = "Install deps, init submodules (agent-skills + mcp), apply patches, wire .env, print status"
+run = "workspace/scripts/setup.sh"
+
+[tasks.status]
+description = "Host repo + submodule + clone state, env keys present, tooling"
+run = "workspace/scripts/status.sh"
+
+[tasks.eval]
+description = "Run evals with the workspace env applied (args pass to `pnpm eval`)"
+run = "workspace/scripts/eval.sh"
+
+[tasks.update]
+description = "Fetch + fast-forward the supabase clone, re-applying patches (-- --check = report only)"
+run = "workspace/scripts/update.sh"
+
+[tasks.publish]
+description = "Clean PR branch from patched-repo work: publish <repo> <topic> [--with <patch>] (or <repo> --list)"
+run = "workspace/scripts/publish.sh"
+
+[tasks.hooks-test]
+description = "Self-test of the pre-push guard lifecycle (install/chain/reinstall; fixtures only, self-cleaning)"
+run = "workspace/scripts/hooks.test.sh"
+
+[tasks.affected]
+description = "Map changed skill/docs/mcp paths to a ready-to-run eval command"
+run = "workspace/scripts/affected-task.sh"
+
+[tasks.ab]
+description = "Head-to-head: <eval> <edited-path> [experiment] — edit applied vs reverted. No args = readiness probe"
+run = "workspace/scripts/ab-task.sh"
+
+[tasks.ab-demo]
+description = "Guided LIVE demo of the docs A/B loop (plants a canary doc, runs the real A/B, cleans up; asks before spending)"
+run = "workspace/scripts/ab-demo.sh"
+
+[tasks.ab-test]
+description = "Zero-cost self-test of the A/B runner (fakes the eval; only touches a clean skill file)"
+run = "workspace/scripts/ab.test.sh"
+
+[tasks.status-test]
+description = "Self-test of status.sh's Next/Ready diagnosis against synthetic workspace states"
+run = "workspace/scripts/status.test.sh"
+
+[tasks.store-key]
+description = "Store an API key in the keychain: store-key <ANTHROPIC_API_KEY|OPENAI_API_KEY|GEMINI_API_KEY>"
+run = "workspace/scripts/store-key.sh"
+
+[tasks.apply-patches]
+description = "(Re)apply the tracked enabler patches into the patched repos; idempotent"
+run = "workspace/scripts/apply-patches.sh"
+
+# --- MCP-server loop (optional; not part of setup) ---
+
+[tasks.mcp-build]
+description = "Build the mcp submodule with the enabler patches applied"
+# Order matters: `git submodule update` on an initialized submodule re-checks
+# out the pin, ORPHANING the patch marker commit (and any user commits) — so
+# init strictly BEFORE apply-patches, and never again after (which is why this
+# does not delegate to the root `pnpm mcp:build`, whose first step is another
+# submodule update; that script is the evals-native unpatched flow).
+run = "git submodule update --init submodules/mcp && pnpm --dir submodules/mcp install && workspace/scripts/apply-patches.sh && pnpm --dir submodules/mcp build"
+
+[tasks.mcp-eval]
+description = "Run evals against the local mcp build (auto init+patch+build)"
+depends = ["mcp-build"]
+run = "workspace/scripts/mcp-eval.sh"
+
+# --- Docs loop (optional, heavy — needs Docker + the supabase CLI) ---
+
+[tasks.clone-docs]
+description = "Sparse-clone supabase/supabase (apps/docs + deps) and install. Run once"
+run = "workspace/scripts/clone-docs.sh"
+
+[tasks.docs-up]
+description = "Start the local content DB (ports 55321+) with docs migrations"
+depends = ["clone-docs"]
+run = "workspace/scripts/apply-patches.sh && workspace/scripts/docs-up.sh"
+
+[tasks.docs-down]
+description = "Stop the local content DB"
+run = "workspace/scripts/docs-down.sh"
+
+[tasks.docs-api]
+description = "Serve the docs content GraphQL API at :3001 for search_docs"
+depends = ["docs-up"]
+run = "workspace/scripts/docs-api.sh"
+
+[tasks.docs-seed]
+description = "Full docs embed — spends OpenAI credits; asks to confirm. Run once"
+depends = ["docs-up"]
+run = "workspace/scripts/docs-seed.sh"
+
+[tasks.docs-index]
+description = "Incremental re-embed of changed docs pages (checksum-based, fail-closed)"
+depends = ["docs-up"]
+run = "workspace/scripts/docs-index.sh"

packages/core/src/index.ts

@@ -906,6 +906,7 @@ export function supabaseMcpServer(
   options: {
     features?: string[];
     version?: string;
+    contentApiUrl?: string;
   } = {}
 ): McpServerDefinition {
   const features = options.features ?? [
@@ -938,6 +939,17 @@ export function supabaseMcpServer(
       if (apiUrl) serverArgs.push('--api-url', apiUrl);
 
       const local = resolveLocalMcpServer();
+      // Alternative docs Content API endpoint (e.g. a locally built docs
+      // index). Only a server that understands --content-api-url can accept
+      // it (the SUPABASE_MCP_SERVER_PATH build; the published 0.8.1 npx
+      // package rejects unknown flags), so the ENV fallback applies only
+      // alongside the local override — a stray env var can't break a plain
+      // npx run. The explicit option is intentional and always honored.
+      const contentApiUrl =
+        options.contentApiUrl ??
+        (local ? process.env.SUPABASE_CONTENT_API_URL : undefined);
+      if (contentApiUrl) serverArgs.push('--content-api-url', contentApiUrl);
+
       if (local) {
         // `node`, not process.execPath: CLI agents run this command INSIDE the
         // sandbox container, where the host's node binary path does not exist.

packages/core/src/mcp-server.test.ts

@@ -27,6 +27,7 @@ import {
 // Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test.
 function clearEnv() {
   vi.stubEnv('SUPABASE_MCP_SERVER_PATH', undefined);
+  vi.stubEnv('SUPABASE_CONTENT_API_URL', undefined);
 }
 
 // A real on-disk build layout: the override path is existence-checked, so the
@@ -57,6 +58,35 @@ describe('supabaseMcpServer().createConfig', () => {
       `@supabase/mcp-server-supabase@${MCP_SERVER_VERSION}`
     );
     expect(config.args).toContain('--api-url');
+    expect(config.args).not.toContain('--content-api-url');
+  });
+
+  it('threads --content-api-url from the env var on the local override path', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir);
+    vi.stubEnv('SUPABASE_CONTENT_API_URL', 'https://env.test/gql');
+    const { config } = await supabaseMcpServer().createConfig({});
+    const i = config.args.indexOf('--content-api-url');
+    expect(i).toBeGreaterThan(-1);
+    expect(config.args[i + 1]).toBe('https://env.test/gql');
+  });
+
+  it('ignores a stray env var on the npx path (0.8.1 rejects unknown flags)', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_CONTENT_API_URL', 'https://env.test/gql');
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.command).toBe('npx');
+    expect(config.args).not.toContain('--content-api-url');
+  });
+
+  it('prefers the explicit contentApiUrl option over the env var', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_CONTENT_API_URL', 'https://env.test/gql');
+    const { config } = await supabaseMcpServer({
+      contentApiUrl: 'https://opt.test/gql',
+    }).createConfig({});
+    const i = config.args.indexOf('--content-api-url');
+    expect(config.args[i + 1]).toBe('https://opt.test/gql');
   });
 
   it('launches a local build dir with node when SUPABASE_MCP_SERVER_PATH is set', async () => {

workspace/README.md

@@ -0,0 +1,87 @@
+# Eval-source workspace
+
+Glue that wires the agent's **inputs** — docs, skills, and the MCP server —
+into this harness, so you can change an input and measure the effect on evals:
+edit a skill, a docs page, or MCP server source, then run the affected evals
+against the local change. Formerly the standalone
+[eval-workspace](https://github.com/supabase/eval-workspace) repo; folded in
+here so the sources under test live beside the harness that tests them
+(direction agreed in the workspace-layout Slack thread, 2026-07-23).
+
+All tasks run via `mise` from the repo root (Supabase convention; `mise.toml`
+lives there). Flag-style args need `--`: `mise run eval -- --eval <id>`.
+
+## Bootstrap
+
+```bash
+mise run status   # the state probe: prints the exact next command for anything missing
+mise run setup    # idempotent: install, init submodules (agent-skills + mcp), patches, .env
+```
+
+Keys live in the macOS keychain as `eval-workspace:<KEY>` — add them with
+`mise run store-key <ANTHROPIC_API_KEY|OPENAI_API_KEY|GEMINI_API_KEY>` (hidden
+prompt, immune to the 128-char truncation of raw `security -w`). Non-macOS:
+put keys in the repo-root `.env` (the fallback `status` will route you to).
+
+`mise run ab-test` is the zero-cost self-check that the glue works.
+
+## The three loops
+
+| Loop | Source | Sync after an edit |
+|---|---|---|
+| Skills | `submodules/agent-skills` (a working tree: edit in place) | none |
+| MCP server | `submodules/mcp` | `mise run mcp-build` |
+| Docs | `supabase/` (opt-in sparse clone: `mise run clone-docs`) | `mise run docs-index` (cents) |
+
+- **MCP loop**: `mise run mcp-eval -- <args>` builds the submodule (with the
+  enabler patches) and runs evals against it via `SUPABASE_MCP_SERVER_PATH`.
+- **Docs loop** (heavy: Docker + supabase CLI): `docs-up` → `docs-seed` (spends
+  OpenAI money once, ~$0.12; always confirm with the user first) → `docs-api`
+  serves the content GraphQL API for `search_docs`; `docs-index` re-embeds
+  changed pages incrementally. To measure a docs edit, point an eval run at
+  the local index through the local server build:
+  `SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql mise run mcp-eval -- --eval <id>`
+  (or just use `mise run ab`, which wires this automatically). Content DB
+  ports are 55321+ to avoid the eval local-stack range (54321-9). Measuring
+  docs impact needs a tools-mode (`interface: mcp`) eval whose answer lives
+  only in the docs.
+
+## Head-to-head A/B
+
+Measure whether ONE edit moves an eval: make a tracked, unstaged edit in a
+loop's scope, then
+
+```bash
+mise run ab <eval-id> <edited-path>   # treatment (edit applied) vs baseline (edit reverted)
+```
+
+The edit is always restored (a failed restore fails the run loudly); per-arm
+provenance receipts land in `results-ab/*.json`. Cost: two model runs. No
+args = readiness probe. First time? `mise run ab-demo` is a guided,
+self-cleaning live proof on the docs loop (spend-gated, asks first).
+
+## Patches & publishing
+
+Local changes to the patched repos (the `supabase/` clone and the
+`submodules/mcp` working tree) are tracked as `.patch` files in
+`workspace/patches/` and applied as marker commits — see
+[patches/README.md](./patches/README.md) for the manifest and the
+publish flow (`mise run publish <repo> <topic>`). A pre-push guard in each
+patched repo blocks marker commits from leaving the machine; the host repo
+never gets hooks or marker commits.
+
+## Provenance
+
+`mise run status -- --json` prints a receipt: host repo SHA + dirty state,
+submodule pins, supabase clone state, patch fingerprints, and the docs-index
+stamp (`.docs-index-stamp.json`, scoped to repo docs content only). `ab.sh`
+embeds a per-arm copy into every A/B result, so a wrong-baseline run is
+immediately obvious.
+
+## Self-tests (all free: no keys, no model spend)
+
+```bash
+mise run status-test   # status.sh Next/Ready diagnosis against synthetic states
+mise run hooks-test    # pre-push guard lifecycle
+mise run ab-test       # A/B runner with a faked eval
+```

workspace/manifest.json

@@ -0,0 +1,28 @@
+{
+  "repos": {
+    "mcp": {
+      "dir": "submodules/mcp",
+      "kind": "submodule",
+      "patches": ["mcp-content-api-url"]
+    },
+    "supabase": {
+      "dir": "supabase",
+      "remote": "git@github.com:supabase/supabase.git",
+      "patches": [
+        "supabase-content-local-ports",
+        "supabase-docs-index-fail-closed",
+        "supabase-docs-guide-checksum",
+        "supabase-docs-lint-warnings-skip",
+        "supabase-docs-reference-dup-sources"
+      ],
+      "localPatches": [
+        "supabase-content-local-ports",
+        "supabase-docs-lint-warnings-skip"
+      ]
+    },
+    "skills": {
+      "dir": "submodules/agent-skills",
+      "kind": "submodule"
+    }
+  }
+}