diff --git a/.gitignore b/.gitignore index 932a6aea..863c53a2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ dist/ results/*/ .sync-tmp/ + +# local-dev runner (apps/framework/scripts/local.ts) +/results-local/ +/.local-docs/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..7c43401e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,112 @@ +# AGENTS.md — supabase/evals + +Instructions for coding agents working in this repo. Humans: start with +[README.md](README.md) and [CONTRIBUTING.md](CONTRIBUTING.md). + +## What this repo is + +Evals for Supabase AI agents. An eval run is `agent + inputs -> score`. The +three inputs a change usually targets: the **skills** tree (in this repo), +the **MCP server** (external checkout), and **docs** content (external +supabase/supabase checkout). + +## Verifying a change against the evals (`pnpm local`) + +Use the local runner for all "did my change help / did it regress?" work. +It never mutates git state, so it is safe alongside in-flight work. + +```bash +pnpm local run [--experiment ] [--runs N] [--mcp ] [--content-api ] +pnpm local compare [same flags] # + diff vs latest published result on origin/main +pnpm local experiments # experiments + which have published baselines +pnpm local docs --docs +``` + +Per input: + +- **Skill edited** (in `skills/`): no sync step — `pnpm local compare `. +- **MCP server edited** (external checkout): `pnpm build` in that checkout, + then `pnpm local compare --mcp `. +- **Docs page edited** (external supabase/supabase checkout): + `pnpm local docs seed --yes` to re-embed (**~$0.12 OpenAI — see spend + rules**), keep `pnpm local docs api` running in a separate terminal, then + `pnpm local compare --content-api http://127.0.0.1:3001/docs/api/graphql --mcp `. + `--content-api` also needs `--mcp`: the env var it sets is only read by an + mcp build carrying supabase/mcp#343 (merged, unreleased). With the published + package `search_docs` would query production docs while the receipt claimed + otherwise, so the runner refuses pre-spend. + **`docs seed` currently fails against a vanilla docs checkout**: the pipeline + unconditionally loads its lint-warnings source, which needs a GitHub App + (`DOCS_GITHUB_APP_*`, no token fallback), and one `Promise.all` makes that + fatal. It aborts before embedding, so a retry costs nothing but achieves + nothing — don't loop on it. The leg needs an index seeded another way until a + skip flag lands upstream. + Score docs evals on retrieval (`docs.calls`, canary content coming back out of + `search_docs`), not on the answer text: tools mode also exposes + `WebSearch`/`WebFetch`, and an edit that contradicts the live page invites the + agent to fetch production and reject the local content as injection (observed). + +Receipts land in `results-local/` (git-ignored): treatment provenance (host +SHA + dirty state, override git state) and, for `compare`, the published +arm's result commit + parent + age. + +## Interpreting results — rules, not suggestions + +- **`compare` is a screen, not causal proof.** The published arm ran in the + scheduled CI world (published MCP package, prod docs index, model state at + refresh time). Never report a flip as caused by the edit; report it as a + signal consistent with the edit. +- **Single runs are noisy.** Before claiming improvement or regression, run + `--runs 3` and read check-level results, not just pass/fail. +- **MCP changes: judge by tool-call activation.** An eval can pass without + ever calling the tool you changed. Confirm the changed tool was actually + exercised (the result JSON records tool calls) before concluding anything. +- **Docs changes: the eval must be able to see the docs.** Use a tools-mode + (`interface: mcp`) eval whose answer lives in the edited page and is + reached via `search_docs`. CLI-scaffold evals can pass regardless of docs. +- **No published baseline?** Use `pnpm local run` (custom evals included). + For a before/after, run once before the edit and once after. + +## Validating a dependency PR (e.g. supabase/mcp) + +1. **Baseline-proof first**: build the dependency's MAIN and run the chosen + eval(s) against it before the PR build — version pins hide fixture drift + (platform-lite tracks the pinned `MCP_SERVER_VERSION`, not your local + build's line; the runner warns on version mismatch). +2. Fixture or eval support living in an unmerged evals PR? Apply it into the + worktree as plain working-tree state: `gh pr diff | git apply`. + Receipts record the dirty tree, so runs stay attributable. +3. Run the PR build with `--mcp `; a main-FAIL -> PR-PASS flip with + everything else constant is a true two-arm comparison on the dependency + axis (stronger than the published screen). +4. **Judge by tool-call activation, not pass/fail**: confirm the changed tool + was called, and unwrap `` envelopes in `toolCalls[]` + before reading results — errors hide inside them. Note that claude-code + records endpoints with an `mcp____` prefix; match with + `.endsWith('')`. + +## Spend rules + +- Eval runs cost model tokens; `pnpm local docs seed` costs **~$0.12 OpenAI + per invocation**. State the cost and get user confirmation before + running paid steps the user did not explicitly request. +- The runner refuses pre-spend on invalid eval metadata, unknown + experiments, and bad `--mcp` paths — do not work around these gates. +- Zero-cost checks: `pnpm --filter @supabase-evals/framework test:local` + (runner self-test), `pnpm local experiments`, `pnpm eval:dry`. + +## Conventions + +- Keys live in `.env` at the repo root: `ANTHROPIC_API_KEY`, plus + `OPENAI_API_KEY` for the docs loop AND for any eval whose scorer uses the + LLM judge (an OpenAI grader model runs even when the agent under test is + Claude). Never hardcode or echo key values. +- Model/agent selection = experiment id. To test an unlisted model, add a + small `experiments/.ts` (copy an existing file's shape) rather than + editing a published experiment in place. +- `results/`, `results-local/`, and `.local-docs/` are outputs — never + commit their contents. +- Verify with `pnpm check` (typecheck + core/sandbox tests) and + `pnpm format:check` (biome) before pushing. +- New evals: follow [CONTRIBUTING.md](CONTRIBUTING.md) (suite choice, + `motivation:` frontmatter, scorer shape). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..5514a3e2 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +See [AGENTS.md](AGENTS.md) for agent instructions in this repo. diff --git a/README.md b/README.md index 7662b01a..681de0f6 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,65 @@ Start the web app development server: pnpm web ``` +## Local development loop (`pnpm local`) + +Testing a change to an agent input — a skill, a local build of +[`mcp-server-supabase`](https://github.com/supabase/mcp), or an edited docs +page — against the evals, without touching git state: + +```bash +pnpm local run [--experiment ] [--mcp ] [--content-api ] +pnpm local compare [same flags] # + diff vs the latest published result on main +pnpm local experiments # list experiments + published-baseline availability +``` + +- **Skills**: edit the skills tree in this repo and just `run` — the harness + reads it as-is. +- **MCP**: clone + build the mcp repo anywhere, then `--mcp ` + (sets `SUPABASE_MCP_SERVER_PATH`, so `search_docs` and friends run your build). +- **Docs**: serve a local docs content API from your own supabase/supabase + checkout, then point runs at it: + + ```bash + pnpm local docs up --docs + pnpm local docs seed # full embed via the docs app's pipeline (~$0.12 OpenAI; asks first) + pnpm local docs api # keep running in a separate terminal + # --content-api needs a local mcp build too: SUPABASE_CONTENT_API_URL support + # is merged (supabase/mcp#343) but unreleased, so the published server ignores + # it and search_docs would silently hit production docs. Refused pre-spend. + pnpm local run --content-api http://127.0.0.1:3001/docs/api/graphql --mcp + ``` + + **Known limitation — `docs seed` needs a docs checkout containing + [supabase/supabase#48364](https://github.com/supabase/supabase/pull/48364).** + Without it, `fetchAllSources()` unconditionally awaits the lint warnings source, + whose loader requires the docs GitHub App, and one shared `Promise.all` turns + that into a full abort before any embedding (so it costs nothing). That PR adds + a token rung below the App, `GH_TOKEN` then `GITHUB_TOKEN`, which is all a + contributor needs: `export GH_TOKEN=$(gh auth token)`. Until it merges, check + that branch out in the checkout you pass to `--docs`. + + Verified end to end against a checkout carrying it, with the `NEXT_PUBLIC_MISC_*` + wiring `docs seed` supplies: the seed completes (1901 sources, 7890 sections) and + a tools-mode eval's `search_docs` returns content that exists only in the local + index. Two rough edges to expect, both upstream: the seed exits 0 while silently + failing 22 `/reference/{javascript,dart}` pages whose sections exceed the + embedding model's 8192-token limit, and a local index has no partner-integration + pages, since that source reads the hosted misc project. Neither blocked the + tested guide-page eval, but an eval whose answer lives in those reference pages + would find them missing from the index. + +Every run writes a provenance receipt to `results-local/` (host SHA + dirty +state, override paths and their git state). `compare` records the published +arm's result commit, parent, and age — and a pass/fail flip against published +is a **screen**, not causal proof: the published run happened in the scheduled +CI world (published mcp package, prod docs index, model state at refresh time). + +Keys go in `.env` at the repo root: `ANTHROPIC_API_KEY`, and `OPENAI_API_KEY` +for the docs loop and for judge-scored evals (the LLM judge is an OpenAI +grader model, regardless of the agent under test). Zero-cost self-test: `pnpm --filter +@supabase-evals/framework test:local`. + ## Eval Shape Every eval contains: diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 0f758e53..19cd9da1 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -32,6 +32,7 @@ import { buildSkillResult, rehydrateTruncatedDocsResults, getExperimentDisplayMetadata, + supabaseMcpServerMounts, } from '@supabase-evals/core'; import type { ExperimentConfig, @@ -439,6 +440,7 @@ async function runOne( // (the session folds the discovery listing into its promptAddendum), // so no skill text is injected into the prompt here. skills: skillSources, + mounts: supabaseMcpServerMounts(), }) ); @@ -507,7 +509,12 @@ async function runOne( // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0). // An in-process agent runs host-side with no sandbox. await using cliSandbox = agentRunsInSandbox - ? disposable(await createBareSandbox({ skills: skillSources })) + ? disposable( + await createBareSandbox({ + skills: skillSources, + mounts: supabaseMcpServerMounts(), + }) + ) : undefined; await using session = disposable( await exp.runtime.startSession({ diff --git a/apps/framework/package.json b/apps/framework/package.json index c726bc37..b047cbf3 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -12,7 +12,9 @@ "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", - "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" + "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts", + "local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/local.ts", + "test:local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-local.ts" }, "dependencies": { "@ai-sdk/anthropic": "catalog:", diff --git a/apps/framework/scripts/docs/content-api-server.ts b/apps/framework/scripts/docs/content-api-server.ts new file mode 100644 index 00000000..d54a5a94 --- /dev/null +++ b/apps/framework/scripts/docs/content-api-server.ts @@ -0,0 +1,71 @@ +// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported +/** + * Standalone docs content GraphQL API for `search_docs`. + * + * Serves the docs app's own route handler (apps/docs/app/api/graphql/route.ts + * in a supabase/supabase checkout) over plain node:http — no Next server. + * Launched by `pnpm local docs api` with the docs checkout's tsx so the + * route's TS + tsconfig conditions resolve; DOCS_ROUTE_PATH points at the + * checkout, PORT picks the listen port. + */ +import { createServer } from 'node:http'; +import { pathToFileURL } from 'node:url'; + +const routePath = process.env.DOCS_ROUTE_PATH; +if (!routePath) { + console.error( + 'DOCS_ROUTE_PATH not set — run this through `pnpm local docs api`' + ); + process.exit(1); +} +// The docs checkout location is user-supplied at runtime; a static import +// cannot name it. +const route = await import(pathToFileURL(routePath).href); +const handlers: Record Promise> = { + GET: route.GET, + OPTIONS: route.OPTIONS, + POST: route.POST, +}; +const port = Number(process.env.PORT ?? 3001); + +createServer(async (incoming, outgoing) => { + const url = new URL( + incoming.url ?? '/', + `http://${incoming.headers.host ?? `127.0.0.1:${port}`}` + ); + const handler = handlers[incoming.method ?? '']; + if (url.pathname !== '/docs/api/graphql' || !handler) { + outgoing.writeHead(404).end(); + return; + } + + const headers = new Headers(); + for (const [name, value] of Object.entries(incoming.headers)) { + if (Array.isArray(value)) + for (const item of value) headers.append(name, item); + else if (value !== undefined) headers.set(name, value); + } + + const chunks: Buffer[] = []; + for await (const chunk of incoming) chunks.push(Buffer.from(chunk)); + const body = + incoming.method === 'GET' || incoming.method === 'HEAD' + ? undefined + : Buffer.concat(chunks).toString('utf8'); + const response = await handler( + new Request(url, { method: incoming.method, headers, body }) + ); + + outgoing.writeHead( + response.status, + Object.fromEntries(response.headers.entries()) + ); + outgoing.end(Buffer.from(await response.arrayBuffer())); + // Bind every interface, advertise loopback — same rule as platform-lite in + // tools mode (see run-eval.ts: sandboxed CLI agents run their MCP servers + // INSIDE the container and reach host-side services via + // host.docker.internal, which arrives on the host's bridge interface, not + // loopback; a 127.0.0.1-only listener refuses those connections). +}).listen(port, '0.0.0.0', () => { + console.log(`Docs content API: http://127.0.0.1:${port}/docs/api/graphql`); +}); diff --git a/apps/framework/scripts/docs/sentry-stub-loader.mjs b/apps/framework/scripts/docs/sentry-stub-loader.mjs new file mode 100644 index 00000000..154e301f --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub-loader.mjs @@ -0,0 +1,11 @@ +// fallow-ignore-file unused-file -- registered at runtime by sentry-stub-register.mjs via module.register() +// Loader-thread resolve hook: '@sentry/nextjs' -> the no-op stub. +const stubUrl = new URL('./sentry-stub.mjs', import.meta.url).href; + +// fallow-ignore-next-line unused-export -- Node loader-hook contract: the module system calls `resolve` +export async function resolve(specifier, context, next) { + if (specifier === '@sentry/nextjs') { + return { url: stubUrl, shortCircuit: true }; + } + return next(specifier, context); +} diff --git a/apps/framework/scripts/docs/sentry-stub-register.mjs b/apps/framework/scripts/docs/sentry-stub-register.mjs new file mode 100644 index 00000000..0c425d1b --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub-register.mjs @@ -0,0 +1,9 @@ +// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported +// Registers a resolve hook that short-circuits '@sentry/nextjs' to the local +// no-op stub. Injected via NODE_OPTIONS from `pnpm local docs api`; chains with tsx's +// own hooks (ours only intercepts the one specifier). Uses module.register() +// (Node 20.6+) rather than registerHooks() (22.15+) — mise pins node "22", +// which an older 22.x install satisfies. +import { register } from 'node:module'; + +register('./sentry-stub-loader.mjs', import.meta.url); diff --git a/apps/framework/scripts/docs/sentry-stub.mjs b/apps/framework/scripts/docs/sentry-stub.mjs new file mode 100644 index 00000000..d42bb83f --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub.mjs @@ -0,0 +1,9 @@ +// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported +// No-op @sentry/nextjs stand-in for the standalone docs content API. +// The route handler calls Sentry.captureException/flush; under plain tsx +// (outside Next's Sentry instrumentation) the real package's ESM build +// resolves without those functions and every request crashes. A local dev +// adapter has no business sending telemetry anyway. Wired up by +// sentry-stub-register.mjs (see local-docs.ts). +export const captureException = () => ''; +export const flush = async () => true; diff --git a/apps/framework/scripts/local-docs.ts b/apps/framework/scripts/local-docs.ts new file mode 100644 index 00000000..7dcb99af --- /dev/null +++ b/apps/framework/scripts/local-docs.ts @@ -0,0 +1,331 @@ +/** + * local-docs.ts — minimal local docs loop for `search_docs` evals. + * + * pnpm local docs up --docs + * pnpm local docs seed # full embed via the docs app's own pipeline (~$0.12 OpenAI; asks first) + * pnpm local docs api [--port N] # serve the content GraphQL API (foreground; keep it running) + * pnpm local docs down + * + * Then point evals at it: + * pnpm local run --content-api http://127.0.0.1:3001/docs/api/graphql + * + * Design: + * - The docs checkout is YOURS (`--docs`), cloned wherever you like — no + * submodule, no patches. Edit pages there, re-seed, re-run. + * - The supabase stack runs from a generated workdir (.local-docs/) with its + * own project id and a port block off both the evals local-stack range + * (54321+) and the docs monorepo default, so it collides with neither. + * Files are COPIED, not symlinked (Windows-safe); `up` regenerates them. + * - Minimal on purpose: full seed only. The upstream pipeline's incremental + * mode has known bugs we found while building the previous iteration + * (guide checksums never set -> guides always re-embed; a skipped source's + * still-valid rows get purged). Incremental lands here once those fixes + * land upstream in supabase/supabase. + * - Some sources need production creds (e.g. DOCS_GITHUB_APP_* for + * lint-warnings); without them the upstream pipeline fails its run. Pass + * them through the environment if you have them. + */ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + cpSync, + existsSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { createInterface } from 'node:readline/promises'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, '..', '..', '..'); +const OVERLAY = join(ROOT, '.local-docs'); +const PROJECT_ID = 'evals-local-docs'; +const DB_CONTAINER = `supabase_db_${PROJECT_ID}`; +// stack ports: whatever block the checkout's config declares -> 443xx (off +// the evals local-stack range 54321-9, and below the macOS ephemeral range +// 49152+, where transient outbound sockets flakily steal listen ports) +const PORT_PREFIX_TO = '443'; +const STACK_EXCLUDES = + 'realtime,storage-api,imgproxy,mailpit,postgres-meta,studio,edge-runtime,logflare,vector,supavisor'; + +const onWindows = process.platform === 'win32'; + +type DocsOptions = { docs?: string; port?: string; yes?: boolean }; + +function fail(msg: string): never { + console.error(msg); + process.exit(1); +} + +/** + * Run a command, streaming output; fails loudly on nonzero exit. + * + * `quiet` buffers instead of streaming, because the supabase CLI reports the + * stack's ANON_KEY, PUBLISHABLE_KEY, SERVICE_ROLE_KEY, SECRET_KEY, and + * JWT_SECRET on every start, plus an update-notifier nag. That buries our own + * one-line status, and the keys are not something to leave on a screen + * recording. + * + * On failure it replays stderr only. Measured against `supabase start`: the key + * report goes to stdout (3 key lines there, 0 on stderr), while stderr carries + * the diagnostics you actually want (workdir, config warnings, per-service + * status). Dropping stdout is a stream boundary rather than a pattern match, so + * there is no redaction regex to keep in step with the CLI's output shapes. + */ +function run( + cmd: string, + args: string[], + opts: { + cwd?: string; + env?: Record; + shim?: boolean; + quiet?: boolean; + } = {} +) { + const res = spawnSync(cmd, args, { + stdio: opts.quiet ? 'pipe' : 'inherit', + encoding: opts.quiet ? 'utf8' : undefined, + cwd: opts.cwd ?? ROOT, + env: opts.env ? { ...process.env, ...opts.env } : process.env, + // .cmd shims (corepack, .bin/tsx) need a shell on Windows + shell: opts.shim ? onWindows : false, + }); + if (res.status !== 0) { + if (opts.quiet && res.stderr) process.stderr.write(res.stderr); + fail(`${cmd} ${args.join(' ')} failed (exit ${res.status})`); + } +} + +/** + * Capture stdout. stderr is swallowed rather than inherited: the supabase CLI + * writes its workdir line, deprecation warnings, stopped-service list, and + * update-notifier nag there on every invocation, and this runs inside helpers + * whose own output is one line. On failure execFileSync throws with the output + * attached, so nothing is lost when it matters. + */ +function capture(cmd: string, args: string[]): string { + return execFileSync(cmd, args, { + cwd: ROOT, + maxBuffer: 1 << 24, + stdio: ['ignore', 'pipe', 'pipe'], + }).toString(); +} + +function docsPath(docs: string | undefined): string { + const marker = join(OVERLAY, 'docs-path.txt'); + let p = + docs ?? + (existsSync(marker) ? readFileSync(marker, 'utf8').trim() : undefined); + if (!p) + fail( + 'no docs checkout configured — pass --docs (git clone https://github.com/supabase/supabase)' + ); + p = isAbsolute(p) ? p : resolve(process.cwd(), p); + if (!existsSync(join(p, 'apps', 'docs'))) + fail(`not a supabase monorepo checkout (apps/docs missing): ${p}`); + return p; +} + +/** Parse `supabase status -o env` output (KEY="value" lines). */ +function stackEnv(): Record { + const out = capture('supabase', [ + 'status', + '--workdir', + OVERLAY, + '-o', + 'env', + ]); + const env: Record = {}; + for (const m of out.matchAll(/^([A-Z_]+)="(.*)"$/gm)) env[m[1]] = m[2]; + if (!env.API_URL) + fail('could not read the local stack env — is it up? (pnpm local docs up)'); + return env; +} + +function cmdUp(opts: DocsOptions) { + const docs = docsPath(opts.docs); + const src = join(docs, 'supabase'); + existsSync(join(src, 'config.toml')) || + fail(`no supabase/config.toml in the docs checkout: ${docs}`); + + // regenerate the overlay workdir: rewritten config + copied stack files + rmSync(OVERLAY, { recursive: true, force: true }); + mkdirSync(join(OVERLAY, 'supabase'), { recursive: true }); + const config = readFileSync(join(src, 'config.toml'), 'utf8') + .replace(/^project_id = ".*"$/m, `project_id = "${PROJECT_ID}"`) + .replace( + /^port = \d{3}(\d{2})$/gm, + (_, tail) => `port = ${PORT_PREFIX_TO}${tail}` + ); + writeFileSync(join(OVERLAY, 'supabase', 'config.toml'), config); + for (const f of ['migrations', 'seed.sql', 'functions', 'buckets']) { + const from = join(src, f); + if (existsSync(from)) + cpSync(from, join(OVERLAY, 'supabase', f), { + recursive: true, + dereference: true, + }); + } + writeFileSync(join(OVERLAY, 'docs-path.txt'), `${docs}\n`); + + console.log(`starting the docs stack (project ${PROJECT_ID})...`); + run('supabase', ['start', '--workdir', OVERLAY, '-x', STACK_EXCLUDES], { + quiet: true, + }); + // Upstream page migrations grant service_role no CRUD on the content + // tables; the embedder authenticates as service_role and needs it. + run( + 'docker', + [ + 'exec', + DB_CONTAINER, + 'psql', + '-U', + 'postgres', + '-d', + 'postgres', + '-q', + '-c', + 'GRANT ALL ON public.page, public.page_section TO service_role; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO service_role; GRANT SELECT ON public.page, public.page_section TO anon, authenticated;', + ], + { quiet: true } + ); + console.log( + `docs stack up on ${stackEnv().API_URL}; next: pnpm local docs seed` + ); +} + +async function cmdSeed(opts: DocsOptions) { + const docs = docsPath(opts.docs); + if (!process.env.OPENAI_API_KEY) + fail('OPENAI_API_KEY not set — add it to .env at the repo root'); + const docsApp = join(docs, 'apps', 'docs'); + if (!existsSync(join(docsApp, 'node_modules'))) { + fail( + `docs app dependencies not installed — run:\n corepack pnpm --dir ${docs} install --filter ./apps/docs...` + ); + } + const env = stackEnv(); + if (!opts.yes && !process.env.LOCAL_DOCS_YES) { + const rl = createInterface({ + input: process.stdin, + output: process.stdout, + }); + const answer = await rl.question( + "Full docs embed: ~1.2M tokens ≈ $0.12 OpenAI. Type 'seed' to proceed: " + ); + rl.close(); + if (answer !== 'seed') fail('cancelled.'); + } + run('corepack', ['pnpm', 'run', 'embeddings:refresh'], { + cwd: docsApp, + shim: true, + env: { + NEXT_PUBLIC_SUPABASE_URL: env.API_URL, + NEXT_PUBLIC_SUPABASE_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY, + SUPABASE_SECRET_KEY: env.SECRET_KEY ?? env.SERVICE_ROLE_KEY, + // generate-embeddings.ts hard-requires these two before doing any work. + // It builds its own client from NEXT_PUBLIC_SUPABASE_URL + + // SUPABASE_SECRET_KEY, but sources/partner-integrations.ts reads the MISC + // pair to pull partner data from the hosted "misc" project. Pointed at the + // local stack they clear the gate; the partner source then finds no such + // tables, which is the right trade for a local docs index. + NEXT_PUBLIC_MISC_URL: env.API_URL, + NEXT_PUBLIC_MISC_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY, + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + NODE_ENV: 'development', + }, + }); + console.log( + 'seeded. next: pnpm local docs api (keep it running in a separate terminal)' + ); +} + +function cmdApi(opts: DocsOptions) { + const docs = docsPath(opts.docs); + const docsApp = join(docs, 'apps', 'docs'); + const port = opts.port ?? '3001'; + const env = stackEnv(); + const tsx = join( + docsApp, + 'node_modules', + '.bin', + onWindows ? 'tsx.cmd' : 'tsx' + ); + if (!existsSync(tsx)) + fail( + `tsx not installed in the docs app — run:\n corepack pnpm --dir ${docs} install --filter ./apps/docs...` + ); + const stub = join(__dirname, 'docs', 'sentry-stub-register.mjs'); + console.log( + `serving on http://127.0.0.1:${port}/docs/api/graphql — point evals at it with --content-api` + ); + // Runs with the DOCS app's tsx + tsconfig so the route's TS and its + // `react-server` condition resolve; the Sentry stub no-ops the route's + // telemetry (the real package crashes outside Next's instrumentation). + run( + tsx, + [ + '--conditions=react-server', + '--tsconfig', + 'tsconfig.json', + join(__dirname, 'docs', 'content-api-server.ts'), + ], + { + cwd: docsApp, + shim: true, + env: { + NODE_ENV: 'development', + PORT: port, + DOCS_ROUTE_PATH: join(docsApp, 'app', 'api', 'graphql', 'route.ts'), + NEXT_PUBLIC_SUPABASE_URL: env.API_URL, + NEXT_PUBLIC_SUPABASE_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY, + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + NODE_OPTIONS: `--import ${stub}${process.env.NODE_OPTIONS ? ` ${process.env.NODE_OPTIONS}` : ''}`, + }, + } + ); +} + +export async function main(argv: string[]) { + const usage = + 'usage: pnpm local docs [--docs ] [--port N] [--yes]'; + const parsed = (() => { + try { + return parseArgs({ + args: argv, + options: { + docs: { type: 'string' }, + port: { type: 'string' }, + yes: { type: 'boolean' }, + }, + allowPositionals: true, + }); + } catch (err) { + fail(`${err instanceof Error ? err.message : String(err)}\n${usage}`); + } + })(); + const { values } = parsed; + switch (parsed.positionals[0]) { + case 'up': + cmdUp(values); + break; + case 'seed': + await cmdSeed(values); + break; + case 'api': + cmdApi(values); + break; + case 'down': + run('supabase', ['stop', '--workdir', OVERLAY], { quiet: true }); + console.log( + `docs stack stopped (project ${PROJECT_ID}); the seeded index stays in its docker volume` + ); + break; + default: + fail(usage); + } +} diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts new file mode 100755 index 00000000..10ec4c61 --- /dev/null +++ b/apps/framework/scripts/local.ts @@ -0,0 +1,647 @@ +#!/usr/bin/env tsx +/** + * local.ts — local-dev runner. Run evals against YOUR inputs (edited skills + * tree, a local MCP build, a custom docs content API) with provenance + * receipts, and optionally compare against the latest published results on + * `origin/main`. + * + * pnpm local run [--experiment ] [--runs N] [--mcp ] [--content-api ] + * pnpm local compare [same flags] + * pnpm local experiments + * pnpm local docs [--docs ] (see local-docs.ts) + * + * Design notes: + * - Treatment-only: nothing here ever mutates a git tree, so concurrent + * sessions/worktrees cannot interfere and in-flight work is never at risk. + * - `compare` is a SCREEN, not causal proof: the published arm ran in the + * scheduled CI world (published MCP package, prod docs index, model state + * at refresh time). The receipt records the published result commit, its + * parent, and its age so the gap is explicit. + * - Explicit over magic: this does not build your MCP checkout or re-embed + * docs for you; it reports what world it measured. Build with + * `pnpm build` in your mcp checkout; serve docs with `pnpm local docs`. + */ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from 'node:fs'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseArgs, type ParseArgsConfig } from 'node:util'; +import { + getExperimentDisplayMetadata, + MCP_SERVER_VERSION, + type ExperimentConfig, +} from '@supabase-evals/core'; +import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; +import { + rawEvalResultSchema, + type RawEvalResult, +} from '@supabase-evals/core/eval-metadata'; +import { main as docsMain } from './local-docs.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, '..', '..', '..'); +// test seam: the smoke suite redirects ALL outputs into a temp sandbox so it +// can never clobber a real (possibly in-flight) run's results/receipts +const RESULTS_ROOT = process.env.LOCAL_RESULTS_ROOT ?? ROOT; +const OUT_DIR = join(RESULTS_ROOT, 'results-local'); +// suite name -> published export file; the values double as the full load list +const PUBLISHED_EXPORTS: Record = { + regression: 'apps/web/src/data/regression-eval-results.json', + benchmark: 'apps/web/src/data/eval-results.json', +}; +const DEFAULT_EXPERIMENT = 'claude-code-sonnet-5'; + +function fail(msg: string): never { + console.error(msg); + process.exit(1); +} + +// ---------- git helpers (plain child_process; cross-platform) ---------- + +function git(args: string[], cwd: string = ROOT): string { + return execFileSync('git', args, { cwd, maxBuffer: 1 << 28 }) + .toString() + .trim(); +} + +function tryGit(args: string[], cwd: string = ROOT): string | undefined { + try { + return git(args, cwd); + } catch { + return undefined; + } +} + +// ---------- provenance receipts ---------- + +type Provenance = { + generatedAt: string; + host: { sha?: string; branch?: string; dirtyFiles: number }; + mcpOverride?: { path: string; sha?: string; dirtyFiles?: number }; + contentApiUrl?: string; + platform: string; +}; + +function collectProvenance(mcpPath?: string, contentApi?: string): Provenance { + const dirty = (cwd: string) => + (tryGit(['status', '--porcelain'], cwd) ?? '').split('\n').filter(Boolean) + .length; + const p: Provenance = { + generatedAt: new Date().toISOString(), + host: { + sha: tryGit(['rev-parse', 'HEAD']), + branch: tryGit(['rev-parse', '--abbrev-ref', 'HEAD']), + dirtyFiles: dirty(ROOT), + }, + platform: `${process.platform}/${process.arch} node ${process.version}`, + }; + if (mcpPath) { + const inRepo = tryGit(['rev-parse', '--show-toplevel'], mcpPath); + p.mcpOverride = { + path: mcpPath, + sha: inRepo ? tryGit(['rev-parse', 'HEAD'], mcpPath) : undefined, + dirtyFiles: inRepo ? dirty(inRepo) : undefined, + }; + } + if (contentApi) p.contentApiUrl = contentApi; + return p; +} + +// ---------- published baselines (compare mode) ---------- + +type PublishedFile = { + file: string; + rows: RawEvalResult[]; + commit: string; + parent: string; + committedAt: string; +}; + +type Baseline = { + row: RawEvalResult; + file: string; + commit: string; + parent: string; + committedAt: string; +}; + +/** Load one published export file from origin/main with its commit metadata. */ +function loadPublishedFile(file: string): PublishedFile | undefined { + let rows: RawEvalResult[]; + try { + rows = JSON.parse(git(['show', `origin/main:${file}`])); + } catch { + return undefined; + } + const [commit, parent, committedAt] = git([ + 'log', + 'origin/main', + '-1', + '--format=%H %P %cI', + '--', + file, + ]).split(' '); + return { file, rows, commit, parent, committedAt }; +} + +/** Fetch origin/main so the published exports are current; warn-and-continue offline. */ +function fetchMain() { + if (process.env.LOCAL_NO_FETCH) return; + try { + git(['fetch', '-q', 'origin', 'main']); + } catch { + console.error( + 'warning: could not fetch origin/main — comparing against the local ref, which may be stale' + ); + } +} + +/** Load every published export once; callers share the result. */ +function loadPublished(): PublishedFile[] { + return Object.values(PUBLISHED_EXPORTS).flatMap( + (f) => loadPublishedFile(f) ?? [] + ); +} + +/** + * Freshest published row per requested eval for the experiment. Refuses + * (pre-spend) when any requested eval has no published row, listing the + * experiments that ARE published for it. + */ +function resolveBaselines( + evalIds: string[], + experiment: string, + files: PublishedFile[] +): Map { + const best = new Map(); + const failures: string[] = []; + for (const id of evalIds) { + const candidates: Baseline[] = files.flatMap( + ({ file, rows, commit, parent, committedAt }) => + rows + .filter((row) => row.eval === id) + .map((row) => ({ row, file, commit, parent, committedAt })) + ); + const match = candidates + .filter((c) => c.row.experiment === experiment) + .sort((a, b) => Date.parse(b.committedAt) - Date.parse(a.committedAt))[0]; + if (match) { + best.set(id, match); + continue; + } + const alts = [...new Set(candidates.map((c) => c.row.experiment))]; + failures.push( + alts.length + ? `no published ${experiment} result for ${id} on origin/main (published experiments: ${alts.join(', ')})` + : `no published result for ${id} on origin/main at all — use \`pnpm local run\` (no baseline needed)` + ); + } + if (failures.length) { + for (const msg of failures) console.error(msg); + process.exit(1); + } + return best; +} + +// ---------- eval validation (fail before spending) ---------- + +function validateEvals(evalIds: string[]) { + for (const id of evalIds) { + const promptPath = join(ROOT, 'evals', id, 'PROMPT.md'); + if (!existsSync(promptPath)) + fail(`no eval at evals/${id} (PROMPT.md missing)`); + try { + parseEvalMarkdown( + readFileSync(promptPath, 'utf8'), + `evals/${id}/PROMPT.md` + ); + } catch (err) { + fail( + `eval metadata invalid — fix evals/${id}/PROMPT.md before spending on runs\n${err instanceof Error ? err.message : String(err)}` + ); + } + } +} + +function validateExperiment(experiment: string) { + if (!existsSync(join(ROOT, 'experiments', `${experiment}.ts`))) { + const available = readdirSync(join(ROOT, 'experiments')) + .filter((f) => f.endsWith('.ts')) + .map((f) => f.replace(/\.ts$/, '')); + fail( + `unknown experiment: ${experiment}\navailable: ${available.join(', ')}\n(or add experiments/${experiment}.ts — see any existing file for the shape)` + ); + } +} + +/** + * The agent itself needs its provider key. Checked here (not just by the + * harness) because the harness SKIPs the experiment with exit 0 on missing + * credentials — the runner would only notice at the no-result check. A + * set-but-EMPTY var counts as missing (node --env-file does not override + * an existing env var, even an empty one, so a stray `export KEY=` in the + * shell silently shadows .env — observed live). + */ +function validateAgentKey() { + if (process.env.ANTHROPIC_API_KEY) return; + fail( + process.env.ANTHROPIC_API_KEY === undefined + ? 'ANTHROPIC_API_KEY not set — add it to .env at the repo root' + : 'ANTHROPIC_API_KEY is set but EMPTY in your shell, which shadows .env (node --env-file never overrides an existing var) — unset it or export a real value' + ); +} + +/** + * Evals whose scorer uses the LLM judge grade the agent's output with an + * OpenAI model — even when the agent under test is Claude. A missing grader + * key otherwise surfaces only AFTER the (paid) agent run, wasting it. + * Textual scan of EVAL.ts; a false positive just asks for a key early. + */ +function validateJudgeKeys(evalIds: string[]) { + if (process.env.OPENAI_API_KEY) return; + const judged = evalIds.filter((id) => { + const scorer = join(ROOT, 'evals', id, 'EVAL.ts'); + return existsSync(scorer) && /\bjudge\b/.test(readFileSync(scorer, 'utf8')); + }); + if (judged.length) + fail( + `these evals score with the LLM judge (OpenAI-backed, regardless of the agent under test): ${judged.join(', ')}\nadd OPENAI_API_KEY to .env at the repo root before running them` + ); +} + +/** + * Accept either the mcp monorepo root or the server package dir for --mcp, + * and refuse pre-spend when the server isn't built (the harness would only + * discover that after eval setup). + */ +function resolveMcpServerPath(raw: string): string { + let p = isAbsolute(raw) ? raw : resolve(process.cwd(), raw); + if (!existsSync(p)) fail(`--mcp path does not exist: ${p}`); + const packageDir = join(p, 'packages', 'mcp-server-supabase'); + if (existsSync(packageDir)) p = packageDir; + if (!existsSync(join(p, 'dist', 'transports', 'stdio.js'))) + fail( + `no built server at ${p} (dist/transports/stdio.js missing) — build it first:\n pnpm install && pnpm build # in the mcp checkout (use \`mise exec --\` if corepack's pnpm mismatches)` + ); + // Fixture-drift heads-up: platform-lite tracks the pinned package version, + // and a local build from a newer line may call endpoints the fixture does + // not serve yet (observed: get_logs moved logs.all -> logs in 0.9.0 while + // the pin and fixture sat at 0.8.x). Warn, don't block. + try { + const local = JSON.parse( + readFileSync(join(p, 'package.json'), 'utf8') + ).version; + if (local && local !== MCP_SERVER_VERSION) + console.error( + `note: local mcp build is v${local}; the harness fixture (platform-lite) tracks the v${MCP_SERVER_VERSION} pin — endpoint drift is possible; judge by tool-call activation, not pass/fail alone` + ); + } catch { + /* unversioned checkout: nothing to compare */ + } + return p; +} + +/** + * `--content-api` is only honoured by a local mcp build. The flag sets + * SUPABASE_CONTENT_API_URL, which the server reads to point `search_docs` at + * a local docs index — support merged in supabase/mcp#343 but shipped in NO + * release yet (newest is v0.9.0; the harness pin is older still). Ungated, + * the published package ignores the var: `search_docs` silently queries + * PRODUCTION docs while collectProvenance still stamps contentApiUrl into the + * receipt — a paid run that measures the wrong world and reports the right + * one. Shape check, so it runs for fake runs too (like resolveMcpServerPath). + */ +function validateContentApi(contentApi: string, mcpServerPath?: string) { + if (!mcpServerPath) + fail( + `--content-api needs --mcp : the published mcp server ignores SUPABASE_CONTENT_API_URL, so search_docs would query production docs while the receipt claims ${contentApi}\n pass --mcp pointing at an mcp checkout on main, built (flag support merged in supabase/mcp#343, not yet released)` + ); + const stdio = join(mcpServerPath, 'dist', 'transports', 'stdio.js'); + if (!readFileSync(stdio, 'utf8').includes('SUPABASE_CONTENT_API_URL')) + fail( + `the mcp build at ${mcpServerPath} predates supabase/mcp#343 and ignores SUPABASE_CONTENT_API_URL — search_docs would query production docs, not ${contentApi}\n update and rebuild the checkout: git pull && pnpm install && pnpm build` + ); +} + +/** + * The experiment's declared skills must exist in this checkout, or the + * treatment silently runs skill-less against a skills-enabled published + * baseline — a world mismatch, not a comparison. + */ +async function validateSkills(experiment: string) { + // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments) + const mod = await import( + pathToFileURL(join(ROOT, 'experiments', `${experiment}.ts`)).href + ); + const skills: string[] = (mod.default as ExperimentConfig).skills ?? []; + const missing = skills.filter((s) => !existsSync(join(ROOT, 'skills', s))); + if (missing.length) + fail( + `experiment ${experiment} declares skills this checkout is missing: ${missing.join(', ')}\ninitialise the skills submodule first: git submodule update --init` + ); +} + +// ---------- treatment run ---------- + +function runEval( + evalId: string, + experiment: string, + runs: number, + env: Record +): string { + const res = spawnSync( + process.execPath, + [ + '--import', + 'tsx/esm', + join(__dirname, '..', 'harness', 'run-eval.ts'), + '--eval', + evalId, + '--experiment', + experiment, + '--runs', + String(runs), + ], + { + stdio: 'inherit', + cwd: join(__dirname, '..'), + env: { ...process.env, ...env }, + } + ); + if (res.status !== 0) fail(`eval run failed: ${evalId} (exit ${res.status})`); + const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`); + if (!existsSync(resultPath)) + fail( + `no result at results/${experiment}/${evalId}.json — check the eval/experiment ids` + ); + return resultPath; +} + +// test hook: LOCAL_EVAL_CMD writes the result file itself (no model spend) +function fakeRun(evalId: string, experiment: string): string { + const resultPath = join( + RESULTS_ROOT, + 'results', + experiment, + `${evalId}.json` + ); + mkdirSync(dirname(resultPath), { recursive: true }); + const res = spawnSync(process.env.LOCAL_EVAL_CMD as string, { + shell: true, + stdio: 'inherit', + env: { ...process.env, RES: resultPath, EVAL: evalId }, + }); + if (res.status !== 0) fail(`LOCAL_EVAL_CMD failed for ${evalId}`); + return resultPath; +} + +// ---------- reporting ---------- + +function reportRow( + label: string, + r: RawEvalResult | undefined, + extra: string +): string { + const checks = r?.checks ?? []; + const checksSummary = `${checks.filter((x) => x.passed).length}/${checks.length}`; + const docsCalls = r?.docs?.calls?.length ?? 0; + return `${label.padEnd(10)} passed=${String(r?.passed).padEnd(5)} checks=${checksSummary.padEnd(6)} docs.calls=${String(docsCalls).padEnd(3)} ${extra}`; +} + +/** Print the published-vs-treatment delta; true when treatment regressed. */ +function reportComparison( + id: string, + b: Baseline, + result: RawEvalResult +): boolean { + writeFileSync( + join(OUT_DIR, `${id}.published.json`), + `${JSON.stringify({ ...b.row, publishedProvenance: { file: b.file, commit: b.commit, parent: b.parent, committedAt: b.committedAt } }, null, 1)}\n` + ); + const ageDays = Math.round( + (Date.now() - new Date(b.committedAt).getTime()) / 86_400_000 + ); + console.log( + reportRow( + 'published', + b.row, + `main@${b.commit.slice(0, 7)} ${b.committedAt.slice(0, 10)} (${ageDays}d old, attempts ${b.row.attempts})` + ) + ); + console.log(reportRow('treatment', result, 'your world')); + const d = (result.passed ? 1 : 0) - (b.row.passed ? 1 : 0); + console.log( + d > 0 + ? '-> IMPROVED vs published (FAIL->PASS)' + : d < 0 + ? '-> REGRESSED vs published (PASS->FAIL)' + : '-> no pass/fail change (compare checks / docs.calls)' + ); + console.log( + 'screen only: the published arm ran in the scheduled CI world — a flip is a signal, not causal proof' + ); + console.log(`saved: results-local/${id}.{published,treatment}.json`); + return d < 0; +} + +/** Run one eval in the treatment world, write its receipt, report; true when it regressed vs published. */ +function runTreatment( + id: string, + experiment: string, + runs: number, + opts: { + env: Record; + mcpPath?: string; + contentApi?: string; + baseline?: Baseline; + } +): boolean { + const { env, mcpPath, contentApi, baseline } = opts; + console.log( + `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}${contentApi ? ', content-api override' : ''}) ==` + ); + const resultPath = process.env.LOCAL_EVAL_CMD + ? fakeRun(id, experiment) + : runEval(id, experiment, runs, env); + + const parsed = rawEvalResultSchema.safeParse( + JSON.parse(readFileSync(resultPath, 'utf8')) + ); + if (!parsed.success) + fail( + `result at ${resultPath} does not match the eval result contract:\n${parsed.error.message}` + ); + const result = parsed.data; + const receipt = { + ...result, + provenance: collectProvenance(mcpPath, contentApi), + }; + writeFileSync( + join(OUT_DIR, `${id}.treatment.json`), + `${JSON.stringify(receipt, null, 1)}\n` + ); + + console.log( + `\n=== local ${baseline ? 'compare' : 'run'}: ${id} (${experiment}) ===` + ); + if (baseline) return reportComparison(id, baseline, result); + console.log(reportRow('treatment', result, 'your world')); + console.log(`saved: results-local/${id}.treatment.json`); + return false; +} + +// ---------- subcommands ---------- + +async function cmdExperiments() { + const published = new Set( + loadPublished().flatMap((f) => f.rows.map((r) => r.experiment)) + ); + console.log( + `${'EXPERIMENT'.padEnd(36)} ${'AGENT'.padEnd(12)} ${'MODEL'.padEnd(22)} ${'EFFORT'.padEnd(8)} PUBLISHED` + ); + for (const f of readdirSync(join(ROOT, 'experiments')) + .filter((f) => f.endsWith('.ts')) + .sort()) { + const name = f.replace(/\.ts$/, ''); + // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments) + const mod = await import(pathToFileURL(join(ROOT, 'experiments', f)).href); + const display = getExperimentDisplayMetadata( + mod.default as ExperimentConfig + ); + console.log( + `${name.padEnd(36)} ${(display.agent ?? '?').padEnd(12)} ${(display.modelId ?? '?').padEnd(22)} ${(display.reasoningEffort ?? '-').padEnd(8)} ${published.has(name) ? 'yes (compare)' : '-'}` + ); + } +} + +/** Expand --suite to every eval the published export carries for the experiment. */ +function expandSuite( + suite: string, + experiment: string, + files: PublishedFile[] +): string[] { + const file = PUBLISHED_EXPORTS[suite]; + if (!file) + fail( + `unknown suite: ${suite} (available: ${Object.keys(PUBLISHED_EXPORTS).join(', ')})` + ); + const rows = files.filter((f) => f.file === file).flatMap((f) => f.rows); + const ids = [ + ...new Set( + rows.filter((r) => r.experiment === experiment).map((r) => r.eval) + ), + ].sort(); + if (!ids.length) + fail( + `no ${suite} rows published for ${experiment} (published experiments: ${[...new Set(rows.map((r) => r.experiment))].join(', ')})` + ); + console.log( + `suite ${suite} for ${experiment}: ${ids.length} evals (one model run each)\n ${ids.join('\n ')}` + ); + return ids; +} + +const RUN_USAGE = + 'usage: pnpm local [--experiment ] [--runs N] [--mcp ] [--content-api ]\n pnpm local compare --suite [same flags] # every eval published for the experiment'; + +async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) { + const parsed = (() => { + try { + return parseArgs({ + args: argv, + options: { + experiment: { type: 'string' }, + runs: { type: 'string' }, + suite: { type: 'string' }, + mcp: { type: 'string' }, + 'content-api': { type: 'string' }, + }, + allowPositionals: true, + }); + } catch (err) { + fail(`${err instanceof Error ? err.message : String(err)}\n${RUN_USAGE}`); + } + })(); + const { values, positionals } = parsed; + const experiment = values.experiment ?? DEFAULT_EXPERIMENT; + validateExperiment(experiment); + let published: PublishedFile[] = []; + if (mode === 'compare') { + fetchMain(); + published = loadPublished(); + } + let evalIds = positionals; + if (values.suite) { + if (mode !== 'compare') + fail( + '--suite expands from the published exports and only makes sense with compare' + ); + if (evalIds.length) fail('pass either eval ids or --suite, not both'); + evalIds = expandSuite(values.suite, experiment, published); + } + if (!evalIds.length) fail(RUN_USAGE); + + const baselines = + mode === 'compare' + ? resolveBaselines(evalIds, experiment, published) + : new Map(); + + validateEvals(evalIds); + // these gates are spend-relevant only for real runs; the test hook fakes them + if (!process.env.LOCAL_EVAL_CMD) { + validateAgentKey(); + await validateSkills(experiment); + validateJudgeKeys(evalIds); + } + + const env: Record = {}; + const mcpPath = values.mcp ? resolveMcpServerPath(values.mcp) : undefined; + if (mcpPath) env.SUPABASE_MCP_SERVER_PATH = mcpPath; + const contentApi = values['content-api']; + if (contentApi) { + validateContentApi(contentApi, mcpPath); + env.SUPABASE_CONTENT_API_URL = contentApi; + } + + mkdirSync(OUT_DIR, { recursive: true }); + let exitCode = 0; + for (const id of evalIds) { + const runs = Number(values.runs ?? baselines.get(id)?.row.attempts ?? 1); + const regressed = runTreatment(id, experiment, runs, { + env, + mcpPath, + contentApi, + baseline: baselines.get(id), + }); + if (regressed) exitCode = 1; + } + process.exit(exitCode); +} + +// ---------- entry ---------- + +const [command, ...rest] = process.argv.slice(2); +switch (command) { + case 'run': + case 'compare': + await cmdRunOrCompare(command, rest); + break; + case 'experiments': + await cmdExperiments(); + break; + case 'docs': + await docsMain(rest); + break; + default: + fail(`usage: pnpm local ... + run run eval(s) in your world (skills tree as-is; --mcp / --content-api overrides) + compare run + diff against the latest published result on origin/main + experiments list experiments (agent, model, effort, published-baseline availability) + docs --docs local docs content API`); +} diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts new file mode 100644 index 00000000..60a5bf05 --- /dev/null +++ b/apps/framework/scripts/smoke-local.ts @@ -0,0 +1,295 @@ +/** + * Zero-cost smoke test for the local-dev runner (scripts/local.ts). + * + * Fakes the eval run via LOCAL_EVAL_CMD (no model spend, no docker) and + * reads REAL published baselines from origin/main (no fetch: LOCAL_NO_FETCH). + * + * pnpm --filter @supabase-evals/framework test:local + */ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(__dirname, '..', '..', '..'); +// every output lands in a disposable sandbox — never the checkout's real +// results/ or results-local/ (an in-flight manual run may own those) +const SANDBOX = mkdtempSync(join(tmpdir(), 'smoke-local-')); +const OUT = join(SANDBOX, 'results-local'); +const EXPERIMENT = 'claude-code-sonnet-5'; + +// a published, currently-existing eval id — resolved dynamically so the test +// doesn't rot when the published set changes +const published = JSON.parse( + execFileSync( + 'git', + ['show', 'origin/main:apps/web/src/data/regression-eval-results.json'], + { cwd: ROOT, maxBuffer: 1 << 28 } + ).toString() +) as Array<{ experiment: string; eval: string }>; +const EVAL = published.find( + (r) => r.experiment === EXPERIMENT && existsSync(join(ROOT, 'evals', r.eval)) +)?.eval; +assert.ok(EVAL, 'no published eval with a local evals/ dir found'); + +// LOCAL_EVAL_CMD contract: write a result JSON to $RES for eval $EVAL. +// A script file sidesteps per-platform shell quoting entirely. +const fakeScript = join(SANDBOX, 'fake-eval.cjs'); +writeFileSync( + fakeScript, + `const fs = require('node:fs'); +const path = require('node:path'); +fs.mkdirSync(path.dirname(process.env.RES), { recursive: true }); +fs.writeFileSync( + process.env.RES, + JSON.stringify({ + eval: process.env.EVAL, + experiment: '${EXPERIMENT}', + passed: true, + checks: [{ name: 'x', passed: true }], + }) +); +` +); +const FAKE = `${JSON.stringify(process.execPath)} ${JSON.stringify(fakeScript)}`; + +function local(args: string[], env: Record = {}) { + const res = spawnSync( + process.execPath, + ['--import', 'tsx/esm', join(__dirname, 'local.ts'), ...args], + { + cwd: join(__dirname, '..'), + encoding: 'utf8', + timeout: 60_000, // a regressed pre-spend gate must never reach a real agent run + env: { + ...process.env, + LOCAL_NO_FETCH: '1', + LOCAL_RESULTS_ROOT: SANDBOX, + LOCAL_EVAL_CMD: FAKE, + FORCE_COLOR: '0', + ...env, + }, + } + ); + return { out: `${res.stdout}\n${res.stderr}`, status: res.status }; +} + +let passed = 0; +function ck(name: string, fn: () => void) { + try { + fn(); + passed++; + } catch (err) { + console.error(`FAIL: ${name}`); + throw err; + } +} + +// --- refusals happen pre-spend, with actionable messages --- +{ + const r = local(['compare', 'no-such-eval-xyz']); + ck('unknown eval refused', () => { + assert.equal(r.status, 1); + assert.match(r.out, /no published result for no-such-eval-xyz/); + }); +} +{ + const r = local(['compare', EVAL, '--experiment', 'bogus-model']); + ck('unknown experiment refused with the available list', () => { + assert.equal(r.status, 1); + assert.match(r.out, /unknown experiment: bogus-model/); + assert.match(r.out, /claude-code-sonnet-5/); + }); +} +{ + const r = local(['run', 'not-an-eval-dir']); + ck('missing eval dir refused', () => { + assert.equal(r.status, 1); + assert.match(r.out, /no eval at evals\/not-an-eval-dir/); + }); +} + +// --- compare: delta table + receipts with published provenance --- +{ + const r = local(['compare', EVAL]); + ck('compare prints both rows and the screen caveat', () => { + assert.equal(r.status, 0); + assert.match(r.out, new RegExp(`=== local compare: ${EVAL}`)); + assert.match(r.out, /published .*main@[0-9a-f]{7}/); + assert.match(r.out, /treatment .*your world/); + assert.match(r.out, /screen only:/); + }); + ck('published receipt carries commit provenance', () => { + const receipt = JSON.parse( + readFileSync(join(OUT, `${EVAL}.published.json`), 'utf8') + ); + assert.match(receipt.publishedProvenance.commit, /^[0-9a-f]{40}$/); + assert.match(receipt.publishedProvenance.parent, /^[0-9a-f]{40}$/); + }); + ck('treatment receipt carries host provenance', () => { + const receipt = JSON.parse( + readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8') + ); + assert.match(receipt.provenance.host.sha, /^[0-9a-f]{40}$/); + assert.equal(typeof receipt.provenance.host.dirtyFiles, 'number'); + }); +} + +// --- run: no baseline required (custom evals), receipt only --- +{ + const r = local(['run', EVAL]); + ck('run works without published baseline machinery', () => { + assert.equal(r.status, 0); + assert.match(r.out, new RegExp(`=== local run: ${EVAL}`)); + assert.doesNotMatch(r.out, /published /); + assert.match(r.out, /saved: results-local\//); + }); +} + +// --- mcp override path validation --- +{ + const r = local(['run', EVAL, '--mcp', '/definitely/not/a/path']); + ck('bad --mcp path refused pre-spend', () => { + assert.equal(r.status, 1); + assert.match(r.out, /--mcp path does not exist/); + }); +} + +// --- mcp override: monorepo root resolves to the server package; unbuilt refused --- +{ + const fake = join(SANDBOX, '.smoke-mcp-checkout'); + const pkg = join(fake, 'packages', 'mcp-server-supabase'); + mkdirSync(join(pkg, 'dist', 'transports'), { recursive: true }); + + const unbuilt = local(['run', EVAL, '--mcp', fake]); + ck('unbuilt mcp checkout refused pre-spend with build hint', () => { + assert.equal(unbuilt.status, 1); + assert.match(unbuilt.out, /no built server at .*mcp-server-supabase/); + assert.match(unbuilt.out, /pnpm install && pnpm build/); + }); + + writeFileSync( + join(pkg, 'dist', 'transports', 'stdio.js'), + '// smoke fixture\n' + ); + const built = local(['run', EVAL, '--mcp', fake]); + ck('monorepo root resolves to the server package dir', () => { + assert.equal(built.status, 0); + const receipt = JSON.parse( + readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8') + ); + assert.match( + receipt.provenance.mcpOverride.path, + /packages[/\\]mcp-server-supabase$/ + ); + }); + + // --- --content-api: refused unless a build that honours it is supplied --- + const noMcp = local(['run', EVAL, '--content-api', 'http://127.0.0.1:3001']); + ck('--content-api without --mcp refused pre-spend', () => { + assert.equal(noMcp.status, 1); + assert.match(noMcp.out, /--content-api needs --mcp/); + assert.match(noMcp.out, /production docs/); + }); + + // the fixture above is a bare stub, i.e. a build predating supabase/mcp#343 + const staleBuild = local([ + 'run', + EVAL, + '--content-api', + 'http://127.0.0.1:3001', + '--mcp', + fake, + ]); + ck('mcp build that ignores the env var refused pre-spend', () => { + assert.equal(staleBuild.status, 1); + assert.match(staleBuild.out, /predates supabase\/mcp#343/); + }); + + writeFileSync( + join(pkg, 'dist', 'transports', 'stdio.js'), + '// smoke fixture reading process.env.SUPABASE_CONTENT_API_URL\n' + ); + const honoured = local([ + 'run', + EVAL, + '--content-api', + 'http://127.0.0.1:3001', + '--mcp', + fake, + ]); + ck('build honouring the env var is accepted and recorded', () => { + assert.equal(honoured.status, 0); + const receipt = JSON.parse( + readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8') + ); + assert.equal(receipt.provenance.contentApiUrl, 'http://127.0.0.1:3001'); + }); + rmSync(fake, { recursive: true, force: true }); +} + +// --- --suite: expands to the published set; guarded against misuse --- +{ + const r = local(['compare', '--suite', 'regression']); + ck('suite expands and runs every published eval', () => { + assert.equal(r.status, 0); + assert.match(r.out, /suite regression for claude-code-sonnet-5: \d+ evals/); + assert.ok( + (r.out.match(/=== local compare: /g) ?? []).length >= 2, + 'expected multiple compare blocks' + ); + }); + const wrongMode = local(['run', '--suite', 'regression']); + ck('suite refused in run mode', () => { + assert.equal(wrongMode.status, 1); + assert.match(wrongMode.out, /only makes sense with compare/); + }); + const both = local(['compare', EVAL, '--suite', 'regression']); + ck('suite plus ids refused', () => { + assert.equal(both.status, 1); + assert.match(both.out, /not both/); + }); + const bogus = local(['compare', '--suite', 'nope']); + ck('unknown suite lists available', () => { + assert.equal(bogus.status, 1); + assert.match(bogus.out, /unknown suite: nope.*regression, benchmark/); + }); +} + +// --- judge-key gate: refused pre-spend, before any agent spawn --- +{ + // needs an eval whose scorer really uses the judge; EVAL may not + const judgedEval = published.find( + (row) => + row.experiment === EXPERIMENT && + existsSync(join(ROOT, 'evals', row.eval, 'EVAL.ts')) && + /\bjudge\b/.test( + readFileSync(join(ROOT, 'evals', row.eval, 'EVAL.ts'), 'utf8') + ) + )?.eval; + assert.ok(judgedEval, 'no judged eval found in the published set'); + const r = local(['run', judgedEval], { + LOCAL_EVAL_CMD: '', + OPENAI_API_KEY: '', + }); + ck('judged eval without OPENAI_API_KEY refused pre-spend', () => { + assert.equal(r.status, 1); + assert.match(r.out, /score with the LLM judge/); + assert.match(r.out, /add OPENAI_API_KEY/); + }); +} + +// cleanup: everything lived in the sandbox +rmSync(SANDBOX, { recursive: true, force: true }); + +console.log(`smoke-local: ${passed} checks passed`); diff --git a/docs/local-workflows-design-brief.md b/docs/local-workflows-design-brief.md new file mode 100644 index 00000000..2743bf7c --- /dev/null +++ b/docs/local-workflows-design-brief.md @@ -0,0 +1,180 @@ +# Design brief: "Test your change against the evals" — single-page HTML + +Audience for this document: a design agent implementing a beautiful, +self-contained, single-page HTML presentation. Everything needed (copy, +structure, data, constraints) is in this brief; no repo access required. + +## Purpose & audience + +One page that teaches a Supabase engineer the three local eval workflows in +under two minutes of scanning: + +> I changed an agent input — a **skill**, the **MCP server**, or a **docs +> page**. How do I verify the change improved an eval, or at least didn't +> regress one? + +Viewers are engineers. They want the mental model, the exact commands, and +the honest limits — in that order. The page is presentation-first (screen +share in a team meeting, then linked in Slack), so it must read well both +projected and self-served. + +## The one mental model (hero concept) + +The page hangs on a single idea — **inputs are explicit overrides**: + +- An eval run is `agent + inputs -> score`. +- The three inputs come from *your* machine, not managed clones: the skills + tree lives in the evals repo; the MCP server is your own checkout passed + via `--mcp`; docs are served from your own supabase/supabase checkout via + `--content-api`. +- One command family drives everything: `pnpm local …`. +- Every run leaves a **provenance receipt** (what world was measured), and + every comparison against published results is a **screen, not causal + proof** (the published arm ran in CI's world at refresh time). + +Suggested hero: the equation/flow rendered visually — + +``` + skills tree ─┐ + --mcp ─┼─> pnpm local run/compare ─> verdict + receipt + --content-api ┘ +``` + +## Page structure (top to bottom) + +1. **Hero**: title + the mental model + the flow graphic. + - Title suggestion: "Test your change against the evals" + - Subtitle: "Three inputs, one command family, honest verdicts." +2. **Decision strip** ("What did you change?") — three buttons/cards that + anchor-link to the workflow sections: Skill · MCP server · Docs page. +3. **Three workflow sections** (content below). Consistent internal layout: + speed/cost badges → 3-4 step commands → "what the verdict means" note. +4. **Shared semantics band**: receipts, pre-spend gates, screen-vs-proof. +5. **Comparison table** (the three loops side by side). +6. **Footer**: links (placeholders): PR #128, README section "Local + development loop", repo. + +## Section content (copy is final; do not rewrite technical strings) + +### Workflow 1 — Skills · badges: `fastest` `$0 extra` + +The skills tree lives in the evals repo itself. Edit and run; the harness +reads it as-is. + +```bash +vim skills/supabase/... # 1. edit the skill +pnpm local compare # 2. run + diff vs the published result +``` + +Note: iteration cost is model spend only. No build step, no services. + +### Workflow 2 — MCP server · badges: `rebuild in seconds` `$0 extra` + +Your own checkout, built locally, passed explicitly. + +```bash +git clone https://github.com/supabase/mcp ~/dev/mcp # once +cd ~/dev/mcp && pnpm install && pnpm build # once + +vim ~/dev/mcp/packages/mcp-server-supabase/src/... # 1. edit +pnpm build # 2. rebuild (seconds) +pnpm local compare --mcp ~/dev/mcp # 3. run + diff +``` + +Note (render as a callout): judge MCP changes by **tool-call activation**, +not pass/fail alone — an eval can pass without ever calling the tool you +changed. The receipt records which tools were called. + +### Workflow 3 — Docs page · badges: `needs Docker` `~$0.12 per re-embed` + +Docs are served from your own supabase/supabase checkout through a local +content API; the eval's `search_docs` reads your index. + +```bash +git clone https://github.com/supabase/supabase ~/dev/supabase # once +pnpm local docs up --docs ~/dev/supabase # once per session +pnpm local docs seed # embed (~$0.12, asks first) +pnpm local docs api # separate terminal, keep running + +vim ~/dev/supabase/apps/docs/content/guides/...mdx # 1. edit +pnpm local docs seed --yes # 2. re-embed (~$0.12) +pnpm local compare \ + --content-api http://127.0.0.1:3001/docs/api/graphql # 3. run + diff +``` + +Note (render as a callout): pick an eval that can *see* the docs — a +tools-mode eval whose answer lives in the edited page. Incremental +re-embeds (cents instead of $0.12) arrive once the upstream pipeline fixes +land. + +### Shared semantics band (applies to all three) + +- **Verdict**: `compare` prints published vs treatment (pass/fail, checks, + docs calls) and one line: IMPROVED · REGRESSED · no change. Nonzero exit + on regression, so it works as a gate. +- **Receipts**: every run writes `results-local/.treatment.json` — + host SHA + dirty state, override paths and their git state; `compare` + adds the published arm's result commit, parent, and age. +- **Screen, not proof** (give this visual weight): the published arm ran in + the scheduled CI world (published MCP package, prod docs index, model + state at refresh time). One run is n=1. Before claiming a number moved: + `--runs 3`, read check-level results, not just pass/fail. +- **Pre-spend gates**: invalid eval metadata, unknown experiment, or a bad + `--mcp` path refuse *before* any model call. +- **No baseline? No problem**: custom evals (not in the published set) use + `pnpm local run` — same receipts, no comparison row. + +### Comparison table + +| | Skills | MCP server | Docs page | +|---|---|---|---| +| Where you edit | `skills/` in the evals repo | your mcp checkout | your supabase/supabase checkout | +| Sync step | none | `pnpm build` (~seconds) | `pnpm local docs seed` (~$0.12) | +| Services needed | none | none | Docker + supabase CLI + docs api terminal | +| Extra flag | — | `--mcp ` | `--content-api ` | +| Iteration cost | model runs only | model runs only | model runs + ~$0.12 embed | +| Judge by | checks | tool-call activation + checks | docs.calls + checks | + +## Visual direction + +- **Supabase brand**: dark theme (near-black background, e.g. #0F0F0F / + #1C1C1C surfaces), Supabase green `#3ECF8E` as THE accent (verdicts, + badges, active states), off-white text. Generous whitespace; feels like + supabase.com, not a wiki. +- Typography: a clean geometric sans for headings (Circular-adjacent; system + fallback fine), high-quality monospace for commands (JetBrains Mono / + ui-monospace). +- Code blocks are first-class citizens: syntax-tinted, copy-to-clipboard + button, step numbers rendered in the gutter (the `# 1.` comments above may + become styled step markers). +- Each workflow gets an icon and an accent tint within the green family; + badges are small pills (speed/cost) at the section top. +- The "screen, not proof" message deserves a distinct visual treatment — an + amber/neutral callout, NOT an error style. It's honesty, not a warning. +- A rendered verdict example adds credibility — mock a small terminal card: + + ``` + === local compare: docs-rls-discovery (claude-code-sonnet-5) === + published passed=false checks=0/1 docs.calls=2 main@ccdacd9 2026-07-25 (0d old) + treatment passed=true checks=1/1 docs.calls=3 your world + -> IMPROVED vs published (FAIL->PASS) + ``` + + (Illustrative output; keep the shape, values may be stylized.) + +## Constraints + +- **Single self-contained `.html` file**: inline CSS + minimal inline JS + (copy buttons, anchor scrolling). No build step, no external JS. System + fonts or one font via CDN link at most; page must degrade gracefully + offline. +- Responsive: presentable projected at 1920w, readable at 375w. +- No screenshots of real dashboards; everything drawn/styled. +- Accessible: real semantic headings, code in `
`, contrast AA.
+- Keep total page weight small (<200KB without fonts).
+
+## Out of scope
+
+- No interactive terminal emulation, no animation beyond subtle hover/entry.
+- Do not invent additional workflows, flags, or costs beyond this brief.
+- Command strings, flag names, paths, and prices are exact — do not edit.
diff --git a/package.json b/package.json
index 08a0ef27..786bb7c3 100644
--- a/package.json
+++ b/package.json
@@ -17,7 +17,8 @@
     "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp",
     "demo:executor": "pnpm --filter @supabase-evals/framework demo:executor",
     "format": "biome check --write . && pnpm --filter @supabase-evals/web format",
-    "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check"
+    "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check",
+    "local": "pnpm --filter @supabase-evals/framework local"
   },
   "dependencies": {
     "@ai-sdk/anthropic": "catalog:",
diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts
index 1a8d541c..323cdf5f 100644
--- a/packages/core/src/eval-metadata.ts
+++ b/packages/core/src/eval-metadata.ts
@@ -306,6 +306,7 @@ const evalResultShape = {
 
 // Raw result files may carry extra fields we don't model; tolerate them.
 export const rawEvalResultSchema = z.looseObject(evalResultShape);
+export type RawEvalResult = z.infer;
 
 // Web-facing result; a clean strict object so its inferred type stays usable.
 export const evalResultSchema = z.object({
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index c3ccfc0e..20d21ac2 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -1,11 +1,17 @@
 import vm from 'node:vm';
 import { createRequire } from 'node:module';
 import { createHash, createHmac } from 'node:crypto';
-import { execFile } from 'node:child_process';
+import { execFile, execFileSync } from 'node:child_process';
 import { createServer } from 'node:net';
 import { promisify } from 'node:util';
-import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
-import { basename, dirname, join } from 'node:path';
+import {
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+} from 'node:fs';
+import { basename, dirname, join, resolve } from 'node:path';
 import { tmpdir } from 'node:os';
 import { fileURLToPath } from 'node:url';
 import type { ToolName } from './transcript/types.js';
@@ -402,6 +408,20 @@ export type AgentHarness = {
  */
 export type SkillSource = { name: string; dir: string };
 
+/**
+ * A host directory bind-mounted into the agent sandbox. Read-only by default;
+ * mounted at the identical container path unless `containerPath` overrides it
+ * (identical paths let one command config work on both host and container).
+ */
+export type SandboxMount = {
+  /** Host directory to mount. */
+  hostPath: string;
+  /** Mount point inside the container; defaults to `hostPath`. */
+  containerPath?: string;
+  /** Mount read-only (default true). */
+  readonly?: boolean;
+};
+
 export type LocalStackSessionArgs = {
   /** Supabase CLI version this scenario requires, overriding the runtime default. */
   cliVersion?: string;
@@ -437,6 +457,12 @@ export type LocalStackSessionArgs = {
    * instead, so they ignore this.
    */
   skills?: readonly SkillSource[];
+  /**
+   * Extra host directories to bind-mount into the sandbox (read-only by
+   * default) — e.g. a local MCP server build the in-container agent must be
+   * able to launch. See `supabaseMcpServerMounts`.
+   */
+  mounts?: readonly SandboxMount[];
 };
 
 /** A mocked hosted project (platform-lite) the sandbox CLI is linked to. */
@@ -895,8 +921,9 @@ export function supabaseMcpServer(
   return {
     name: 'supabase-mcp',
     async createConfig({ apiUrl, accessToken } = {}) {
-      const args = [
-        `@supabase/mcp-server-supabase@${version}`,
+      // Server flags are identical whether we launch the published package via
+      // npx or a local build directly with node.
+      const serverArgs = [
         // The server refuses to boot without a token; with only platform-
         // independent features (docs) it never authenticates against the
         // management API, so a well-formed throwaway is enough.
@@ -908,12 +935,122 @@ export function supabaseMcpServer(
       // Only point the server at a platform when one is given. `docs` is
       // platform-independent (it queries the public docs GraphQL API), so a
       // docs-only server runs standalone with no `--api-url`.
-      if (apiUrl) args.push('--api-url', apiUrl);
-      return { config: { command: 'npx', args } };
+      if (apiUrl) serverArgs.push('--api-url', apiUrl);
+
+      // Docs override: point `search_docs` at a local content API instead of
+      // the public docs GraphQL. Baked into args rather than left to the parent
+      // environment because CLI agents spawn this command INSIDE the sandbox
+      // container, which inherits nothing from the harness process — and
+      // `rewriteLoopback` then maps 127.0.0.1 -> host.docker.internal so the
+      // host-side API is actually reachable from in there. Flag support landed
+      // in supabase/mcp#343 (unreleased), so this needs a local build; `pnpm
+      // local` refuses --content-api without one.
+      const contentApiUrl = process.env.SUPABASE_CONTENT_API_URL;
+      if (contentApiUrl) serverArgs.push('--content-api-url', contentApiUrl);
+
+      const local = resolveLocalMcpServer();
+      if (local) {
+        // `node`, not process.execPath: CLI agents run this command INSIDE the
+        // sandbox container, where the host's node binary path does not exist.
+        // Both container and host resolve `node` via PATH.
+        return {
+          config: { command: 'node', args: [local.entry, ...serverArgs] },
+        };
+      }
+
+      return {
+        config: {
+          command: 'npx',
+          args: [`@supabase/mcp-server-supabase@${version}`, ...serverArgs],
+        },
+      };
     },
   };
 }
 
+/**
+ * SUPABASE_MCP_SERVER_PATH swaps the published npx package for a local build
+ * (a repo/package dir or a direct .js/.mjs/.cjs entrypoint), so a workspace
+ * can test an unpublished server change without publishing to npm. Relative
+ * paths resolve against the evals checkout root (not the process CWD), so
+ * `submodules/mcp/packages/mcp-server-supabase` works from any directory.
+ *
+ * Memoized per env value: createConfig and the sandbox mounts both resolve,
+ * and each resolution spawns git (anchor + mount root) — cache so repeat
+ * calls within a run cost nothing. Keyed on the raw env string because tests
+ * (and in principle callers) change it between calls; the not-found error
+ * path is deliberately uncached so a fixed build is picked up on retry.
+ */
+type LocalMcpServer = { entry: string; baseDir: string; mountRoot: string };
+let localMcpServerCache: { key: string; value: LocalMcpServer } | null = null;
+
+function resolveLocalMcpServer(): LocalMcpServer | null {
+  const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH;
+  if (!localServerPath) return null;
+  if (localMcpServerCache?.key === localServerPath)
+    return localMcpServerCache.value;
+
+  const anchor =
+    gitToplevel(dirname(fileURLToPath(import.meta.url))) ?? process.cwd();
+  const isEntryFile = /\.[cm]?js$/.test(localServerPath);
+  const base = resolve(anchor, localServerPath);
+  const probe = isEntryFile
+    ? base
+    : join(base, 'dist', 'transports', 'stdio.js');
+  if (!existsSync(probe)) {
+    throw new Error(
+      `SUPABASE_MCP_SERVER_PATH resolved to ${probe}, which does not exist — ` +
+        `build the server first (pnpm install && pnpm build in the mcp checkout); ` +
+        `see README "Running against an exact MCP server revision".`
+    );
+  }
+  // One filesystem view for command AND mount: the sandbox bind-mounts the
+  // realpath (Docker resolves sources against the daemon's view), so the
+  // command must reference the same view — an override under a symlinked dir
+  // (macOS /tmp -> /private/tmp) would otherwise exec a path that does not
+  // exist in-container. Canonicalize the BASE once and derive the entry from
+  // it (never realpath the entry separately: a symlinked dist/ target could
+  // resolve outside the mounted baseDir).
+  const realBase = realpathSync(base);
+  const baseDir = isEntryFile ? dirname(realBase) : realBase;
+  const value: LocalMcpServer = {
+    entry: isEntryFile
+      ? realBase
+      : join(realBase, 'dist', 'transports', 'stdio.js'),
+    baseDir,
+    // The whole git toplevel (not just dist/) because the build is unbundled:
+    // it requires its node_modules at runtime.
+    mountRoot: gitToplevel(baseDir) ?? baseDir,
+  };
+  localMcpServerCache = { key: localServerPath, value };
+  return value;
+}
+
+function gitToplevel(dir: string): string | null {
+  try {
+    return execFileSync('git', ['rev-parse', '--show-toplevel'], {
+      cwd: dir,
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Sandbox mounts required to launch the SUPABASE_MCP_SERVER_PATH build inside
+ * a containerized agent sandbox. A CLI agent's MCP command runs INSIDE the
+ * container, where the host build is invisible — so the build's checkout is
+ * bind-mounted read-only at its identical (real) path, letting the same
+ * config work on both sides, with host rebuilds visible immediately (no
+ * re-copy). Empty when unset.
+ */
+export function supabaseMcpServerMounts(): SandboxMount[] {
+  const local = resolveLocalMcpServer();
+  return local ? [{ hostPath: local.mountRoot, readonly: true }] : [];
+}
+
 export function executorMcpServer(): McpServerDefinition {
   return {
     name: 'executor-mcp',
diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts
new file mode 100644
index 00000000..9f30397a
--- /dev/null
+++ b/packages/core/src/mcp-server.test.ts
@@ -0,0 +1,190 @@
+import {
+  afterAll,
+  afterEach,
+  beforeAll,
+  describe,
+  expect,
+  it,
+  vi,
+} from 'vitest';
+import { execFileSync } from 'node:child_process';
+import {
+  mkdirSync,
+  mkdtempSync,
+  realpathSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { join, relative } from 'node:path';
+import { tmpdir } from 'node:os';
+import { rewriteLoopback } from './agents/shared.js';
+import {
+  MCP_SERVER_VERSION,
+  supabaseMcpServer,
+  supabaseMcpServerMounts,
+} from './index.js';
+
+// Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test.
+// SUPABASE_CONTENT_API_URL is cleared too: it now adds server args, so an
+// ambient value would leak into every assertion below.
+function clearEnv() {
+  vi.stubEnv('SUPABASE_MCP_SERVER_PATH', undefined);
+  vi.stubEnv('SUPABASE_CONTENT_API_URL', undefined);
+}
+
+// A real on-disk build layout: the override path is existence-checked, so the
+// fixtures must actually exist for the happy paths (and not for the error one).
+let fixtureDir: string;
+let fixtureEntry: string;
+beforeAll(() => {
+  // realpath'd: the resolver realpaths the override (command must match the
+  // container mount view), so unresolved tmpdir paths (macOS /var symlink)
+  // would fail every exact-path assertion below.
+  fixtureDir = realpathSync(mkdtempSync(join(tmpdir(), 'mcp-override-')));
+  fixtureEntry = join(fixtureDir, 'dist', 'transports', 'stdio.js');
+  mkdirSync(join(fixtureDir, 'dist', 'transports'), { recursive: true });
+  writeFileSync(fixtureEntry, '');
+});
+afterAll(() => rmSync(fixtureDir, { recursive: true, force: true }));
+
+describe('supabaseMcpServer().createConfig', () => {
+  afterEach(() => vi.unstubAllEnvs());
+
+  it('defaults to the published package via npx', async () => {
+    clearEnv();
+    const { config } = await supabaseMcpServer().createConfig({
+      apiUrl: 'http://api.test',
+    });
+    expect(config.command).toBe('npx');
+    expect(config.args[0]).toBe(
+      `@supabase/mcp-server-supabase@${MCP_SERVER_VERSION}`
+    );
+    expect(config.args).toContain('--api-url');
+  });
+
+  it('launches a local build dir with node when SUPABASE_MCP_SERVER_PATH is set', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir);
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.command).toBe('node');
+    expect(config.args[0]).toBe(fixtureEntry);
+  });
+
+  it('uses a direct .js override path as-is', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureEntry);
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.args[0]).toBe(fixtureEntry);
+  });
+
+  it('preserves --api-url on the local override path', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir);
+    const { config } = await supabaseMcpServer().createConfig({
+      apiUrl: 'http://api.test',
+    });
+    const i = config.args.indexOf('--api-url');
+    expect(i).toBeGreaterThan(-1);
+    expect(config.args[i + 1]).toBe('http://api.test');
+  });
+
+  it('fails fast with an actionable error when the override path does not exist', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', join(fixtureDir, 'not-built'));
+    await expect(supabaseMcpServer().createConfig({})).rejects.toThrow(
+      /does not exist.*build the server first/s
+    );
+  });
+  it('resolves a relative override path against the evals checkout root', async () => {
+    clearEnv();
+    const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
+      cwd: process.cwd(),
+      encoding: 'utf8',
+    }).trim();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', relative(repoRoot, fixtureEntry));
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.args[0]).toBe(fixtureEntry);
+  });
+
+  it('realpaths a symlinked override so the command matches the container mount', async () => {
+    clearEnv();
+    const linkDir = mkdtempSync(join(tmpdir(), 'mcp-link-'));
+    const link = join(linkDir, 'pkg');
+    symlinkSync(fixtureDir, link);
+    try {
+      vi.stubEnv('SUPABASE_MCP_SERVER_PATH', link);
+      const { config } = await supabaseMcpServer().createConfig({});
+      expect(config.args[0]).toBe(fixtureEntry); // the real path, not the symlink
+      expect(supabaseMcpServerMounts()).toEqual([
+        { hostPath: realpathSync(fixtureDir), readonly: true },
+      ]);
+    } finally {
+      rmSync(linkDir, { recursive: true, force: true });
+    }
+  });
+
+  it('passes SUPABASE_CONTENT_API_URL as --content-api-url so it survives into the sandbox', async () => {
+    clearEnv();
+    vi.stubEnv(
+      'SUPABASE_CONTENT_API_URL',
+      'http://127.0.0.1:3001/docs/api/graphql'
+    );
+    const { config } = await supabaseMcpServer().createConfig({});
+    // In args, not env: a CLI agent spawns this inside the container, which
+    // inherits nothing from the harness process.
+    const flag = config.args.indexOf('--content-api-url');
+    expect(flag).toBeGreaterThan(-1);
+    expect(config.args[flag + 1]).toBe(
+      'http://127.0.0.1:3001/docs/api/graphql'
+    );
+
+    // ...and being in args is what lets the container reach the host-side API.
+    const rewritten = rewriteLoopback({ supabase: config });
+    expect(rewritten.supabase.args).toContain(
+      'http://host.docker.internal:3001/docs/api/graphql'
+    );
+  });
+
+  it('omits the flag when no local docs API is configured', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_CONTENT_API_URL', undefined);
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.args).not.toContain('--content-api-url');
+  });
+});
+
+describe('supabaseMcpServerMounts', () => {
+  afterEach(() => vi.unstubAllEnvs());
+
+  it('is empty when no override is set', () => {
+    clearEnv();
+    expect(supabaseMcpServerMounts()).toEqual([]);
+  });
+  it("mounts the override checkout root read-only (a CLI agent's MCP command runs in-container)", () => {
+    clearEnv();
+    // A git checkout wrapping the package dir: the mount must cover the whole
+    // checkout (the unbundled build needs its node_modules), not just dist/.
+    const checkout = realpathSync(mkdtempSync(join(tmpdir(), 'mcp-mount-')));
+    try {
+      execFileSync('git', ['init', '-q'], { cwd: checkout });
+      const pkgDir = join(checkout, 'packages', 'server');
+      mkdirSync(join(pkgDir, 'dist', 'transports'), { recursive: true });
+      writeFileSync(join(pkgDir, 'dist', 'transports', 'stdio.js'), '');
+      vi.stubEnv('SUPABASE_MCP_SERVER_PATH', pkgDir);
+      expect(supabaseMcpServerMounts()).toEqual([
+        { hostPath: checkout, readonly: true },
+      ]);
+    } finally {
+      rmSync(checkout, { recursive: true, force: true });
+    }
+  });
+
+  it('falls back to the package dir when the override is not inside a git checkout', () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir);
+    expect(supabaseMcpServerMounts()).toEqual([
+      { hostPath: realpathSync(fixtureDir), readonly: true },
+    ]);
+  });
+});
diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts
index 4d046395..b17918f5 100644
--- a/packages/sandbox/src/agent-environment.ts
+++ b/packages/sandbox/src/agent-environment.ts
@@ -13,7 +13,7 @@
  * this builder, so adding/removing an environment component happens in one place.
  */
 
-import type { SkillSource } from '@supabase-evals/core';
+import type { SandboxMount, SkillSource } from '@supabase-evals/core';
 import { DockerSandbox } from './docker-sandbox.js';
 import {
   ensureSupabaseSandboxImage,
@@ -43,6 +43,12 @@ export interface AgentEnvironmentOptions {
    * mode. This is the only difference between the two environments.
    */
   localStack?: LocalStackSetup;
+  /**
+   * Extra host directories bind-mounted into the sandbox (read-only by
+   * default) — e.g. a local MCP server build the in-container agent must be
+   * able to launch.
+   */
+  mounts?: readonly SandboxMount[];
 }
 
 export interface AgentEnvironment {
@@ -69,6 +75,7 @@ export async function createAgentEnvironment(
     // stack and instead reaches host-side platform-lite over the default bridge
     // via host.docker.internal — so bridge there.
     network: options.localStack ? 'host' : undefined,
+    mounts: options.mounts,
   });
   try {
     if (options.localStack) {
diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts
index 0ea025ef..4c51a14d 100644
--- a/packages/sandbox/src/bare-sandbox.ts
+++ b/packages/sandbox/src/bare-sandbox.ts
@@ -1,4 +1,8 @@
-import type { AgentSandbox, SkillSource } from '@supabase-evals/core';
+import type {
+  AgentSandbox,
+  SandboxMount,
+  SkillSource,
+} from '@supabase-evals/core';
 import { createAgentEnvironment } from './agent-environment.js';
 import { toAgentSandbox } from './local-stack-runtime.js';
 import { buildSkillsPrompt } from './skills.js';
@@ -20,11 +24,16 @@ export interface BareSandboxHandle {
  * platform-lite via `host.docker.internal` on the default bridge).
  */
 export async function createBareSandbox(
-  options: { cliVersion?: string; skills?: readonly SkillSource[] } = {}
+  options: {
+    cliVersion?: string;
+    skills?: readonly SkillSource[];
+    mounts?: readonly SandboxMount[];
+  } = {}
 ): Promise {
   const env = await createAgentEnvironment({
     cliVersion: options.cliVersion,
     skills: options.skills,
+    mounts: options.mounts,
   });
   return {
     sandbox: toAgentSandbox(env.sandbox),
diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts
index 42f98052..586956ca 100644
--- a/packages/sandbox/src/docker-sandbox.ts
+++ b/packages/sandbox/src/docker-sandbox.ts
@@ -16,6 +16,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { promisify } from 'node:util';
+import type { SandboxMount } from '@supabase-evals/core';
 import type { SandboxCommandResult } from './types.js';
 
 const execFileAsync = promisify(execFile);
@@ -78,6 +79,13 @@ export interface DockerSandboxOptions {
    * Omitted means Docker's default bridge.
    */
   network?: string;
+  /**
+   * Extra host directories bind-mounted into the container (read-only unless
+   * a mount sets `readonly: false`), at the identical path unless
+   * `containerPath` overrides it. Used to expose host artifacts the agent's
+   * tools must execute — e.g. a local MCP server build.
+   */
+  mounts?: readonly SandboxMount[];
 }
 
 export interface RunCommandOptions {
@@ -90,6 +98,7 @@ export class DockerSandbox {
   private defaultTimeoutMs: number;
   private network: string | undefined;
   private image: string;
+  private mounts: readonly SandboxMount[];
   readonly workdir: string;
   /**
    * Env vars injected into every `runShell` (non-root) command — both the
@@ -102,6 +111,7 @@ export class DockerSandbox {
     this.defaultTimeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
     this.network = options.network;
     this.image = options.image ?? DEFAULT_IMAGE;
+    this.mounts = options.mounts ?? [];
     this.workdir = `${WORKSPACE_BASE}-${randomUUID().slice(0, 8)}`;
   }
 
@@ -134,6 +144,14 @@ export class DockerSandbox {
         '/var/run/docker.sock:/var/run/docker.sock',
         '--volume',
         `${this.workdir}:${this.workdir}`,
+        // Caller-requested host mounts (e.g. a local MCP server build the
+        // in-container agent launches). Read-only unless the mount opts out.
+        ...this.mounts.flatMap((mount) => [
+          '--volume',
+          `${mount.hostPath}:${mount.containerPath ?? mount.hostPath}${
+            mount.readonly === false ? '' : ':ro'
+          }`,
+        ]),
         '--workdir',
         this.workdir,
         // Reach host-side servers (e.g. the linked platform-lite) at
diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts
index 5fe61875..6ee95e72 100644
--- a/packages/sandbox/src/local-stack-runtime.ts
+++ b/packages/sandbox/src/local-stack-runtime.ts
@@ -79,6 +79,7 @@ export function localStackRuntime(
       projectRunning,
       hosted,
       skills,
+      mounts,
     }) {
       // Local-stack mode = the shared agent environment with the Supabase local
       // stack started. Everything else (image, tooling, skills) is identical to
@@ -87,6 +88,7 @@ export function localStackRuntime(
         cliVersion: cliVersion ?? options.cliVersion,
         localDir,
         skills,
+        mounts,
         localStack: {
           includeServices,
           projectRunning,