From bb9be5775df0cd09ff7b50a398c09e117482a855 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Mon, 27 Jul 2026 09:24:01 +0200 Subject: [PATCH 01/24] feat: local MCP server override (SUPABASE_MCP_SERVER_PATH) Re-lands the core piece of #118 without the submodule: createConfig resolves a local MCP build (path canonicalized against the evals checkout root, symlinked dist/ supported, mount fallback for containerized agents) when SUPABASE_MCP_SERVER_PATH is set. Includes the review fixes from the #118 thread (canonicalize base once, receipt-visible mounts) and the full test suite (13 mcp-server tests). --- apps/framework/harness/run-eval.ts | 9 +- packages/core/src/index.ts | 140 ++++++++++++++++- packages/core/src/mcp-server.test.ts | 157 ++++++++++++++++++++ packages/sandbox/src/agent-environment.ts | 9 +- packages/sandbox/src/bare-sandbox.ts | 13 +- packages/sandbox/src/docker-sandbox.ts | 18 +++ packages/sandbox/src/local-stack-runtime.ts | 2 + 7 files changed, 337 insertions(+), 11 deletions(-) create mode 100644 packages/core/src/mcp-server.test.ts diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 0f758e53..19cd9da1 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -32,6 +32,7 @@ import { buildSkillResult, rehydrateTruncatedDocsResults, getExperimentDisplayMetadata, + supabaseMcpServerMounts, } from '@supabase-evals/core'; import type { ExperimentConfig, @@ -439,6 +440,7 @@ async function runOne( // (the session folds the discovery listing into its promptAddendum), // so no skill text is injected into the prompt here. skills: skillSources, + mounts: supabaseMcpServerMounts(), }) ); @@ -507,7 +509,12 @@ async function runOne( // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0). // An in-process agent runs host-side with no sandbox. await using cliSandbox = agentRunsInSandbox - ? disposable(await createBareSandbox({ skills: skillSources })) + ? disposable( + await createBareSandbox({ + skills: skillSources, + mounts: supabaseMcpServerMounts(), + }) + ) : undefined; await using session = disposable( await exp.runtime.startSession({ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c3ccfc0e..e25fc37b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,11 +1,17 @@ import vm from 'node:vm'; import { createRequire } from 'node:module'; import { createHash, createHmac } from 'node:crypto'; -import { execFile } from 'node:child_process'; +import { execFile, execFileSync } from 'node:child_process'; import { createServer } from 'node:net'; import { promisify } from 'node:util'; -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { basename, dirname, join } from 'node:path'; +import { + existsSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, +} from 'node:fs'; +import { basename, dirname, join, resolve } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import type { ToolName } from './transcript/types.js'; @@ -402,6 +408,20 @@ export type AgentHarness = { */ export type SkillSource = { name: string; dir: string }; +/** + * A host directory bind-mounted into the agent sandbox. Read-only by default; + * mounted at the identical container path unless `containerPath` overrides it + * (identical paths let one command config work on both host and container). + */ +export type SandboxMount = { + /** Host directory to mount. */ + hostPath: string; + /** Mount point inside the container; defaults to `hostPath`. */ + containerPath?: string; + /** Mount read-only (default true). */ + readonly?: boolean; +}; + export type LocalStackSessionArgs = { /** Supabase CLI version this scenario requires, overriding the runtime default. */ cliVersion?: string; @@ -437,6 +457,12 @@ export type LocalStackSessionArgs = { * instead, so they ignore this. */ skills?: readonly SkillSource[]; + /** + * Extra host directories to bind-mount into the sandbox (read-only by + * default) — e.g. a local MCP server build the in-container agent must be + * able to launch. See `supabaseMcpServerMounts`. + */ + mounts?: readonly SandboxMount[]; }; /** A mocked hosted project (platform-lite) the sandbox CLI is linked to. */ @@ -895,8 +921,9 @@ export function supabaseMcpServer( return { name: 'supabase-mcp', async createConfig({ apiUrl, accessToken } = {}) { - const args = [ - `@supabase/mcp-server-supabase@${version}`, + // Server flags are identical whether we launch the published package via + // npx or a local build directly with node. + const serverArgs = [ // The server refuses to boot without a token; with only platform- // independent features (docs) it never authenticates against the // management API, so a well-formed throwaway is enough. @@ -908,12 +935,111 @@ export function supabaseMcpServer( // Only point the server at a platform when one is given. `docs` is // platform-independent (it queries the public docs GraphQL API), so a // docs-only server runs standalone with no `--api-url`. - if (apiUrl) args.push('--api-url', apiUrl); - return { config: { command: 'npx', args } }; + if (apiUrl) serverArgs.push('--api-url', apiUrl); + + const local = resolveLocalMcpServer(); + if (local) { + // `node`, not process.execPath: CLI agents run this command INSIDE the + // sandbox container, where the host's node binary path does not exist. + // Both container and host resolve `node` via PATH. + return { + config: { command: 'node', args: [local.entry, ...serverArgs] }, + }; + } + + return { + config: { + command: 'npx', + args: [`@supabase/mcp-server-supabase@${version}`, ...serverArgs], + }, + }; }, }; } +/** + * SUPABASE_MCP_SERVER_PATH swaps the published npx package for a local build + * (a repo/package dir or a direct .js/.mjs/.cjs entrypoint), so a workspace + * can test an unpublished server change without publishing to npm. Relative + * paths resolve against the evals checkout root (not the process CWD), so + * `submodules/mcp/packages/mcp-server-supabase` works from any directory. + * + * Memoized per env value: createConfig and the sandbox mounts both resolve, + * and each resolution spawns git (anchor + mount root) — cache so repeat + * calls within a run cost nothing. Keyed on the raw env string because tests + * (and in principle callers) change it between calls; the not-found error + * path is deliberately uncached so a fixed build is picked up on retry. + */ +type LocalMcpServer = { entry: string; baseDir: string; mountRoot: string }; +let localMcpServerCache: { key: string; value: LocalMcpServer } | null = null; + +function resolveLocalMcpServer(): LocalMcpServer | null { + const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH; + if (!localServerPath) return null; + if (localMcpServerCache?.key === localServerPath) + return localMcpServerCache.value; + + const anchor = + gitToplevel(dirname(fileURLToPath(import.meta.url))) ?? process.cwd(); + const isEntryFile = /\.[cm]?js$/.test(localServerPath); + const base = resolve(anchor, localServerPath); + const probe = isEntryFile + ? base + : join(base, 'dist', 'transports', 'stdio.js'); + if (!existsSync(probe)) { + throw new Error( + `SUPABASE_MCP_SERVER_PATH resolved to ${probe}, which does not exist — ` + + `build the server first (pnpm install && pnpm build in the mcp checkout); ` + + `see README "Running against an exact MCP server revision".` + ); + } + // One filesystem view for command AND mount: the sandbox bind-mounts the + // realpath (Docker resolves sources against the daemon's view), so the + // command must reference the same view — an override under a symlinked dir + // (macOS /tmp -> /private/tmp) would otherwise exec a path that does not + // exist in-container. Canonicalize the BASE once and derive the entry from + // it (never realpath the entry separately: a symlinked dist/ target could + // resolve outside the mounted baseDir). + const realBase = realpathSync(base); + const baseDir = isEntryFile ? dirname(realBase) : realBase; + const value: LocalMcpServer = { + entry: isEntryFile + ? realBase + : join(realBase, 'dist', 'transports', 'stdio.js'), + baseDir, + // The whole git toplevel (not just dist/) because the build is unbundled: + // it requires its node_modules at runtime. + mountRoot: gitToplevel(baseDir) ?? baseDir, + }; + localMcpServerCache = { key: localServerPath, value }; + return value; +} + +function gitToplevel(dir: string): string | null { + try { + return execFileSync('git', ['rev-parse', '--show-toplevel'], { + cwd: dir, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + } catch { + return null; + } +} + +/** + * Sandbox mounts required to launch the SUPABASE_MCP_SERVER_PATH build inside + * a containerized agent sandbox. A CLI agent's MCP command runs INSIDE the + * container, where the host build is invisible — so the build's checkout is + * bind-mounted read-only at its identical (real) path, letting the same + * config work on both sides, with host rebuilds visible immediately (no + * re-copy). Empty when unset. + */ +export function supabaseMcpServerMounts(): SandboxMount[] { + const local = resolveLocalMcpServer(); + return local ? [{ hostPath: local.mountRoot, readonly: true }] : []; +} + export function executorMcpServer(): McpServerDefinition { return { name: 'executor-mcp', diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts new file mode 100644 index 00000000..bd3f81c3 --- /dev/null +++ b/packages/core/src/mcp-server.test.ts @@ -0,0 +1,157 @@ +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from 'vitest'; +import { execFileSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { join, relative } from 'node:path'; +import { tmpdir } from 'node:os'; +import { + MCP_SERVER_VERSION, + supabaseMcpServer, + supabaseMcpServerMounts, +} from './index.js'; + +// Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test. +function clearEnv() { + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', undefined); +} + +// A real on-disk build layout: the override path is existence-checked, so the +// fixtures must actually exist for the happy paths (and not for the error one). +let fixtureDir: string; +let fixtureEntry: string; +beforeAll(() => { + // realpath'd: the resolver realpaths the override (command must match the + // container mount view), so unresolved tmpdir paths (macOS /var symlink) + // would fail every exact-path assertion below. + fixtureDir = realpathSync(mkdtempSync(join(tmpdir(), 'mcp-override-'))); + fixtureEntry = join(fixtureDir, 'dist', 'transports', 'stdio.js'); + mkdirSync(join(fixtureDir, 'dist', 'transports'), { recursive: true }); + writeFileSync(fixtureEntry, ''); +}); +afterAll(() => rmSync(fixtureDir, { recursive: true, force: true })); + +describe('supabaseMcpServer().createConfig', () => { + afterEach(() => vi.unstubAllEnvs()); + + it('defaults to the published package via npx', async () => { + clearEnv(); + const { config } = await supabaseMcpServer().createConfig({ + apiUrl: 'http://api.test', + }); + expect(config.command).toBe('npx'); + expect(config.args[0]).toBe( + `@supabase/mcp-server-supabase@${MCP_SERVER_VERSION}` + ); + expect(config.args).toContain('--api-url'); + }); + + it('launches a local build dir with node when SUPABASE_MCP_SERVER_PATH is set', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.command).toBe('node'); + expect(config.args[0]).toBe(fixtureEntry); + }); + + it('uses a direct .js override path as-is', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureEntry); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); + }); + + it('preserves --api-url on the local override path', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir); + const { config } = await supabaseMcpServer().createConfig({ + apiUrl: 'http://api.test', + }); + const i = config.args.indexOf('--api-url'); + expect(i).toBeGreaterThan(-1); + expect(config.args[i + 1]).toBe('http://api.test'); + }); + + it('fails fast with an actionable error when the override path does not exist', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', join(fixtureDir, 'not-built')); + await expect(supabaseMcpServer().createConfig({})).rejects.toThrow( + /does not exist.*build the server first/s + ); + }); + it('resolves a relative override path against the evals checkout root', async () => { + clearEnv(); + const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], { + cwd: process.cwd(), + encoding: 'utf8', + }).trim(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', relative(repoRoot, fixtureEntry)); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); + }); + + it('realpaths a symlinked override so the command matches the container mount', async () => { + clearEnv(); + const linkDir = mkdtempSync(join(tmpdir(), 'mcp-link-')); + const link = join(linkDir, 'pkg'); + symlinkSync(fixtureDir, link); + try { + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', link); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.args[0]).toBe(fixtureEntry); // the real path, not the symlink + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: realpathSync(fixtureDir), readonly: true }, + ]); + } finally { + rmSync(linkDir, { recursive: true, force: true }); + } + }); +}); + +describe('supabaseMcpServerMounts', () => { + afterEach(() => vi.unstubAllEnvs()); + + it('is empty when no override is set', () => { + clearEnv(); + expect(supabaseMcpServerMounts()).toEqual([]); + }); + it("mounts the override checkout root read-only (a CLI agent's MCP command runs in-container)", () => { + clearEnv(); + // A git checkout wrapping the package dir: the mount must cover the whole + // checkout (the unbundled build needs its node_modules), not just dist/. + const checkout = realpathSync(mkdtempSync(join(tmpdir(), 'mcp-mount-'))); + try { + execFileSync('git', ['init', '-q'], { cwd: checkout }); + const pkgDir = join(checkout, 'packages', 'server'); + mkdirSync(join(pkgDir, 'dist', 'transports'), { recursive: true }); + writeFileSync(join(pkgDir, 'dist', 'transports', 'stdio.js'), ''); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', pkgDir); + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: checkout, readonly: true }, + ]); + } finally { + rmSync(checkout, { recursive: true, force: true }); + } + }); + + it('falls back to the package dir when the override is not inside a git checkout', () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir); + expect(supabaseMcpServerMounts()).toEqual([ + { hostPath: realpathSync(fixtureDir), readonly: true }, + ]); + }); +}); diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index 4d046395..b17918f5 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -13,7 +13,7 @@ * this builder, so adding/removing an environment component happens in one place. */ -import type { SkillSource } from '@supabase-evals/core'; +import type { SandboxMount, SkillSource } from '@supabase-evals/core'; import { DockerSandbox } from './docker-sandbox.js'; import { ensureSupabaseSandboxImage, @@ -43,6 +43,12 @@ export interface AgentEnvironmentOptions { * mode. This is the only difference between the two environments. */ localStack?: LocalStackSetup; + /** + * Extra host directories bind-mounted into the sandbox (read-only by + * default) — e.g. a local MCP server build the in-container agent must be + * able to launch. + */ + mounts?: readonly SandboxMount[]; } export interface AgentEnvironment { @@ -69,6 +75,7 @@ export async function createAgentEnvironment( // stack and instead reaches host-side platform-lite over the default bridge // via host.docker.internal — so bridge there. network: options.localStack ? 'host' : undefined, + mounts: options.mounts, }); try { if (options.localStack) { diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index 0ea025ef..4c51a14d 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -1,4 +1,8 @@ -import type { AgentSandbox, SkillSource } from '@supabase-evals/core'; +import type { + AgentSandbox, + SandboxMount, + SkillSource, +} from '@supabase-evals/core'; import { createAgentEnvironment } from './agent-environment.js'; import { toAgentSandbox } from './local-stack-runtime.js'; import { buildSkillsPrompt } from './skills.js'; @@ -20,11 +24,16 @@ export interface BareSandboxHandle { * platform-lite via `host.docker.internal` on the default bridge). */ export async function createBareSandbox( - options: { cliVersion?: string; skills?: readonly SkillSource[] } = {} + options: { + cliVersion?: string; + skills?: readonly SkillSource[]; + mounts?: readonly SandboxMount[]; + } = {} ): Promise { const env = await createAgentEnvironment({ cliVersion: options.cliVersion, skills: options.skills, + mounts: options.mounts, }); return { sandbox: toAgentSandbox(env.sandbox), diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts index 42f98052..586956ca 100644 --- a/packages/sandbox/src/docker-sandbox.ts +++ b/packages/sandbox/src/docker-sandbox.ts @@ -16,6 +16,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { promisify } from 'node:util'; +import type { SandboxMount } from '@supabase-evals/core'; import type { SandboxCommandResult } from './types.js'; const execFileAsync = promisify(execFile); @@ -78,6 +79,13 @@ export interface DockerSandboxOptions { * Omitted means Docker's default bridge. */ network?: string; + /** + * Extra host directories bind-mounted into the container (read-only unless + * a mount sets `readonly: false`), at the identical path unless + * `containerPath` overrides it. Used to expose host artifacts the agent's + * tools must execute — e.g. a local MCP server build. + */ + mounts?: readonly SandboxMount[]; } export interface RunCommandOptions { @@ -90,6 +98,7 @@ export class DockerSandbox { private defaultTimeoutMs: number; private network: string | undefined; private image: string; + private mounts: readonly SandboxMount[]; readonly workdir: string; /** * Env vars injected into every `runShell` (non-root) command — both the @@ -102,6 +111,7 @@ export class DockerSandbox { this.defaultTimeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; this.network = options.network; this.image = options.image ?? DEFAULT_IMAGE; + this.mounts = options.mounts ?? []; this.workdir = `${WORKSPACE_BASE}-${randomUUID().slice(0, 8)}`; } @@ -134,6 +144,14 @@ export class DockerSandbox { '/var/run/docker.sock:/var/run/docker.sock', '--volume', `${this.workdir}:${this.workdir}`, + // Caller-requested host mounts (e.g. a local MCP server build the + // in-container agent launches). Read-only unless the mount opts out. + ...this.mounts.flatMap((mount) => [ + '--volume', + `${mount.hostPath}:${mount.containerPath ?? mount.hostPath}${ + mount.readonly === false ? '' : ':ro' + }`, + ]), '--workdir', this.workdir, // Reach host-side servers (e.g. the linked platform-lite) at diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 5fe61875..6ee95e72 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -79,6 +79,7 @@ export function localStackRuntime( projectRunning, hosted, skills, + mounts, }) { // Local-stack mode = the shared agent environment with the Supabase local // stack started. Everything else (image, tooling, skills) is identical to @@ -87,6 +88,7 @@ export function localStackRuntime( cliVersion: cliVersion ?? options.cliVersion, localDir, skills, + mounts, localStack: { includeServices, projectRunning, From e636f50e6452713a3cfd580ec683684d6d798b3e Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Mon, 27 Jul 2026 09:36:55 +0200 Subject: [PATCH 02/24] feat: TypeScript local-dev runner (pnpm local) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cross-platform, treatment-only workflow for running evals against local inputs — no bash, no macOS keychain, no git mutation, no submodules: pnpm local run [--experiment ] [--runs N] [--mcp ] [--content-api ] pnpm local compare [same] # + diff vs the latest published result on origin/main pnpm local experiments # agent/model/effort + published-baseline availability pnpm local docs --docs # minimal local docs content API - run/compare execute the harness with explicit input overrides (local MCP build via SUPABASE_MCP_SERVER_PATH, docs via SUPABASE_CONTENT_API_URL) and write provenance receipts (host SHA + dirty state, mcp override git state, content-api URL) to results-local/. compare records the published arm's result commit, parent SHA, and age, and says explicitly that a flip is a screen, not causal proof. - pre-spend gates: eval metadata parsed via the harness's own parseEvalMarkdown, experiment existence checked, --mcp path validated — all before any model call. - docs loop is deliberately minimal: full seed only, from YOUR supabase/supabase checkout (--docs), run through the docs app's own pipeline into a generated .local-docs workdir (own project id, 443xx ports — off the evals local-stack range and below the macOS ephemeral range). Incremental re-embed waits for the upstream pipeline fixes. - smoke test (test:local): 8 checks, zero model/docker cost. Verified live end to end: docs stack boots from the generated workdir, the standalone content API serves the docs route under tsx with the Sentry no-op stub, and searchDocs returns ranked results against a seeded index. --- .gitignore | 4 + apps/framework/package.json | 82 ++-- .../scripts/docs/content-api-server.ts | 65 +++ .../scripts/docs/sentry-stub-loader.mjs | 9 + .../scripts/docs/sentry-stub-register.mjs | 8 + apps/framework/scripts/docs/sentry-stub.mjs | 8 + apps/framework/scripts/local-docs.ts | 278 +++++++++++ apps/framework/scripts/local.ts | 446 ++++++++++++++++++ apps/framework/scripts/smoke-local.ts | 150 ++++++ package.json | 80 ++-- 10 files changed, 1052 insertions(+), 78 deletions(-) create mode 100644 apps/framework/scripts/docs/content-api-server.ts create mode 100644 apps/framework/scripts/docs/sentry-stub-loader.mjs create mode 100644 apps/framework/scripts/docs/sentry-stub-register.mjs create mode 100644 apps/framework/scripts/docs/sentry-stub.mjs create mode 100644 apps/framework/scripts/local-docs.ts create mode 100755 apps/framework/scripts/local.ts create mode 100644 apps/framework/scripts/smoke-local.ts diff --git a/.gitignore b/.gitignore index 932a6aea..863c53a2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ dist/ results/*/ .sync-tmp/ + +# local-dev runner (apps/framework/scripts/local.ts) +/results-local/ +/.local-docs/ diff --git a/apps/framework/package.json b/apps/framework/package.json index c726bc37..92fbe5b8 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -1,42 +1,44 @@ { - "name": "@supabase-evals/framework", - "private": true, - "version": "0.0.1", - "type": "module", - "scripts": { - "check": "pnpm typecheck && pnpm test:framework", - "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", - "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", - "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", - "typecheck": "tsc --noEmit", - "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", - "export-results": "node --import tsx/esm scripts/export-results.ts", - "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", - "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" - }, - "dependencies": { - "@ai-sdk/anthropic": "catalog:", - "@ai-sdk/mcp": "catalog:", - "@ai-sdk/openai": "catalog:", - "@supabase-evals/platform-lite": "workspace:*", - "@electric-sql/pglite": "catalog:", - "@supabase-evals/core": "workspace:*", - "@supabase-evals/sandbox": "workspace:*", - "@supabase/supabase-js": "catalog:", - "@testing-library/jest-dom": "^6.9.1", - "@testing-library/react": "^16.3.2", - "@vitejs/plugin-react": "catalog:", - "ai": "catalog:", - "happy-dom": "^20.9.0", - "@supabase/lite": "catalog:", - "react": "^19.2.5", - "react-dom": "^19.2.5", - "vite": "catalog:", - "vitest": "catalog:" - }, - "devDependencies": { - "@types/node": "catalog:", - "tsx": "^4.19.0", - "typescript": "catalog:" - } + "name": "@supabase-evals/framework", + "private": true, + "version": "0.0.1", + "type": "module", + "scripts": { + "check": "pnpm typecheck && pnpm test:framework", + "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", + "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", + "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", + "typecheck": "tsc --noEmit", + "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", + "export-results": "node --import tsx/esm scripts/export-results.ts", + "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", + "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts", + "local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/local.ts", + "test:local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-local.ts" + }, + "dependencies": { + "@ai-sdk/anthropic": "catalog:", + "@ai-sdk/mcp": "catalog:", + "@ai-sdk/openai": "catalog:", + "@supabase-evals/platform-lite": "workspace:*", + "@electric-sql/pglite": "catalog:", + "@supabase-evals/core": "workspace:*", + "@supabase-evals/sandbox": "workspace:*", + "@supabase/supabase-js": "catalog:", + "@testing-library/jest-dom": "^6.9.1", + "@testing-library/react": "^16.3.2", + "@vitejs/plugin-react": "catalog:", + "ai": "catalog:", + "happy-dom": "^20.9.0", + "@supabase/lite": "catalog:", + "react": "^19.2.5", + "react-dom": "^19.2.5", + "vite": "catalog:", + "vitest": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsx": "^4.19.0", + "typescript": "catalog:" + } } diff --git a/apps/framework/scripts/docs/content-api-server.ts b/apps/framework/scripts/docs/content-api-server.ts new file mode 100644 index 00000000..f0adbada --- /dev/null +++ b/apps/framework/scripts/docs/content-api-server.ts @@ -0,0 +1,65 @@ +/** + * Standalone docs content GraphQL API for `search_docs`. + * + * Serves the docs app's own route handler (apps/docs/app/api/graphql/route.ts + * in a supabase/supabase checkout) over plain node:http — no Next server. + * Launched by `pnpm local docs api` with the docs checkout's tsx so the + * route's TS + tsconfig conditions resolve; DOCS_ROUTE_PATH points at the + * checkout, PORT picks the listen port. + */ +import { createServer } from 'node:http'; +import { pathToFileURL } from 'node:url'; + +const routePath = process.env.DOCS_ROUTE_PATH; +if (!routePath) { + console.error( + 'DOCS_ROUTE_PATH not set — run this through `pnpm local docs api`' + ); + process.exit(1); +} +// The docs checkout location is user-supplied at runtime; a static import +// cannot name it. +const route = await import(pathToFileURL(routePath).href); +const handlers: Record Promise> = { + GET: route.GET, + OPTIONS: route.OPTIONS, + POST: route.POST, +}; +const port = Number(process.env.PORT ?? 3001); + +createServer(async (incoming, outgoing) => { + const url = new URL( + incoming.url ?? '/', + `http://${incoming.headers.host ?? `127.0.0.1:${port}`}` + ); + const handler = handlers[incoming.method ?? '']; + if (url.pathname !== '/docs/api/graphql' || !handler) { + outgoing.writeHead(404).end(); + return; + } + + const headers = new Headers(); + for (const [name, value] of Object.entries(incoming.headers)) { + if (Array.isArray(value)) + for (const item of value) headers.append(name, item); + else if (value !== undefined) headers.set(name, value); + } + + const chunks: Buffer[] = []; + for await (const chunk of incoming) chunks.push(Buffer.from(chunk)); + const body = + incoming.method === 'GET' || incoming.method === 'HEAD' + ? undefined + : Buffer.concat(chunks).toString('utf8'); + const response = await handler( + new Request(url, { method: incoming.method, headers, body }) + ); + + outgoing.writeHead( + response.status, + Object.fromEntries(response.headers.entries()) + ); + outgoing.end(Buffer.from(await response.arrayBuffer())); +}).listen(port, '127.0.0.1', () => { + console.log(`Docs content API: http://127.0.0.1:${port}/docs/api/graphql`); +}); diff --git a/apps/framework/scripts/docs/sentry-stub-loader.mjs b/apps/framework/scripts/docs/sentry-stub-loader.mjs new file mode 100644 index 00000000..da2d7b19 --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub-loader.mjs @@ -0,0 +1,9 @@ +// Loader-thread resolve hook: '@sentry/nextjs' -> the no-op stub. +const stubUrl = new URL('./sentry-stub.mjs', import.meta.url).href; + +export async function resolve(specifier, context, next) { + if (specifier === '@sentry/nextjs') { + return { url: stubUrl, shortCircuit: true }; + } + return next(specifier, context); +} diff --git a/apps/framework/scripts/docs/sentry-stub-register.mjs b/apps/framework/scripts/docs/sentry-stub-register.mjs new file mode 100644 index 00000000..9d4bc57c --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub-register.mjs @@ -0,0 +1,8 @@ +// Registers a resolve hook that short-circuits '@sentry/nextjs' to the local +// no-op stub. Injected via NODE_OPTIONS from `pnpm local docs api`; chains with tsx's +// own hooks (ours only intercepts the one specifier). Uses module.register() +// (Node 20.6+) rather than registerHooks() (22.15+) — mise pins node "22", +// which an older 22.x install satisfies. +import { register } from 'node:module'; + +register('./sentry-stub-loader.mjs', import.meta.url); diff --git a/apps/framework/scripts/docs/sentry-stub.mjs b/apps/framework/scripts/docs/sentry-stub.mjs new file mode 100644 index 00000000..6eb3c60f --- /dev/null +++ b/apps/framework/scripts/docs/sentry-stub.mjs @@ -0,0 +1,8 @@ +// No-op @sentry/nextjs stand-in for the standalone docs content API. +// The route handler calls Sentry.captureException/flush; under plain tsx +// (outside Next's Sentry instrumentation) the real package's ESM build +// resolves without those functions and every request crashes. A local dev +// adapter has no business sending telemetry anyway. Wired up by +// sentry-stub-register.mjs (see local-docs.ts). +export const captureException = () => ''; +export const flush = async () => true; diff --git a/apps/framework/scripts/local-docs.ts b/apps/framework/scripts/local-docs.ts new file mode 100644 index 00000000..a52c3a0a --- /dev/null +++ b/apps/framework/scripts/local-docs.ts @@ -0,0 +1,278 @@ +/** + * local-docs.ts — minimal local docs loop for `search_docs` evals. + * + * pnpm local docs up --docs + * pnpm local docs seed # full embed via the docs app's own pipeline (~$0.12 OpenAI; asks first) + * pnpm local docs api [--port N] # serve the content GraphQL API (foreground; keep it running) + * pnpm local docs down + * + * Then point evals at it: + * pnpm local run --content-api http://127.0.0.1:3001/docs/api/graphql + * + * Design: + * - The docs checkout is YOURS (`--docs`), cloned wherever you like — no + * submodule, no patches. Edit pages there, re-seed, re-run. + * - The supabase stack runs from a generated workdir (.local-docs/) with its + * own project id and a port block off both the evals local-stack range + * (54321+) and the docs monorepo default, so it collides with neither. + * Files are COPIED, not symlinked (Windows-safe); `up` regenerates them. + * - Minimal on purpose: full seed only. The upstream pipeline's incremental + * mode has known bugs we found while building the previous iteration + * (guide checksums never set -> guides always re-embed; a skipped source's + * still-valid rows get purged). Incremental lands here once those fixes + * land upstream in supabase/supabase. + * - Some sources need production creds (e.g. DOCS_GITHUB_APP_* for + * lint-warnings); without them the upstream pipeline fails its run. Pass + * them through the environment if you have them. + */ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + cpSync, + existsSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { createInterface } from 'node:readline/promises'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, '..', '..', '..'); +const OVERLAY = join(ROOT, '.local-docs'); +const PROJECT_ID = 'evals-local-docs'; +const DB_CONTAINER = `supabase_db_${PROJECT_ID}`; +// stack ports: whatever block the checkout's config declares -> 443xx (off +// the evals local-stack range 54321-9, and below the macOS ephemeral range +// 49152+, where transient outbound sockets flakily steal listen ports) +const PORT_PREFIX_TO = '443'; +const STACK_EXCLUDES = + 'realtime,storage-api,imgproxy,mailpit,postgres-meta,studio,edge-runtime,logflare,vector,supavisor'; + +const onWindows = process.platform === 'win32'; + +function fail(msg: string): never { + console.error(msg); + process.exit(1); +} + +/** Run a command, streaming output; fails loudly on nonzero exit. */ +function run( + cmd: string, + args: string[], + opts: { + cwd?: string; + env?: Record; + shim?: boolean; + } = {} +) { + const res = spawnSync(cmd, args, { + stdio: 'inherit', + cwd: opts.cwd ?? ROOT, + env: opts.env ? { ...process.env, ...opts.env } : process.env, + // .cmd shims (corepack, .bin/tsx) need a shell on Windows + shell: opts.shim ? onWindows : false, + }); + if (res.status !== 0) + fail(`${cmd} ${args.join(' ')} failed (exit ${res.status})`); +} + +function capture(cmd: string, args: string[]): string { + return execFileSync(cmd, args, { cwd: ROOT, maxBuffer: 1 << 24 }).toString(); +} + +function docsPath(flags: Map): string { + const marker = join(OVERLAY, 'docs-path.txt'); + let p = + flags.get('docs') ?? + (existsSync(marker) ? readFileSync(marker, 'utf8').trim() : undefined); + if (!p) + fail( + 'no docs checkout configured — pass --docs (git clone https://github.com/supabase/supabase)' + ); + p = isAbsolute(p) ? p : resolve(process.cwd(), p); + if (!existsSync(join(p, 'apps', 'docs'))) + fail(`not a supabase monorepo checkout (apps/docs missing): ${p}`); + return p; +} + +/** Parse `supabase status -o env` output (KEY="value" lines). */ +function stackEnv(): Record { + const out = capture('supabase', [ + 'status', + '--workdir', + OVERLAY, + '-o', + 'env', + ]); + const env: Record = {}; + for (const m of out.matchAll(/^([A-Z_]+)="(.*)"$/gm)) env[m[1]] = m[2]; + if (!env.API_URL) + fail('could not read the local stack env — is it up? (pnpm local docs up)'); + return env; +} + +function cmdUp(flags: Map) { + const docs = docsPath(flags); + const src = join(docs, 'supabase'); + existsSync(join(src, 'config.toml')) || + fail(`no supabase/config.toml in the docs checkout: ${docs}`); + + // regenerate the overlay workdir: rewritten config + copied stack files + rmSync(OVERLAY, { recursive: true, force: true }); + mkdirSync(join(OVERLAY, 'supabase'), { recursive: true }); + const config = readFileSync(join(src, 'config.toml'), 'utf8') + .replace(/^project_id = ".*"$/m, `project_id = "${PROJECT_ID}"`) + .replace( + /^port = \d{3}(\d{2})$/gm, + (_, tail) => `port = ${PORT_PREFIX_TO}${tail}` + ); + writeFileSync(join(OVERLAY, 'supabase', 'config.toml'), config); + for (const f of ['migrations', 'seed.sql', 'functions', 'buckets']) { + const from = join(src, f); + if (existsSync(from)) + cpSync(from, join(OVERLAY, 'supabase', f), { + recursive: true, + dereference: true, + }); + } + writeFileSync(join(OVERLAY, 'docs-path.txt'), `${docs}\n`); + + run('supabase', ['start', '--workdir', OVERLAY, '-x', STACK_EXCLUDES]); + // Upstream page migrations grant service_role no CRUD on the content + // tables; the embedder authenticates as service_role and needs it. + run('docker', [ + 'exec', + DB_CONTAINER, + 'psql', + '-U', + 'postgres', + '-d', + 'postgres', + '-q', + '-c', + 'GRANT ALL ON public.page, public.page_section TO service_role; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO service_role; GRANT SELECT ON public.page, public.page_section TO anon, authenticated;', + ]); + console.log( + `docs stack up (project ${PROJECT_ID}); next: pnpm local docs seed` + ); +} + +async function cmdSeed(flags: Map) { + const docs = docsPath(flags); + if (!process.env.OPENAI_API_KEY) + fail('OPENAI_API_KEY not set — add it to .env at the repo root'); + const docsApp = join(docs, 'apps', 'docs'); + if (!existsSync(join(docsApp, 'node_modules'))) { + fail( + `docs app dependencies not installed — run:\n corepack pnpm --dir ${docs} install --filter ./apps/docs...` + ); + } + const env = stackEnv(); + if (!flags.has('yes') && !process.env.LOCAL_DOCS_YES) { + const rl = createInterface({ + input: process.stdin, + output: process.stdout, + }); + const answer = await rl.question( + "Full docs embed: ~1.2M tokens ≈ $0.12 OpenAI. Type 'seed' to proceed: " + ); + rl.close(); + if (answer !== 'seed') fail('cancelled.'); + } + run('corepack', ['pnpm', 'run', 'embeddings:refresh'], { + cwd: docsApp, + shim: true, + env: { + NEXT_PUBLIC_SUPABASE_URL: env.API_URL, + NEXT_PUBLIC_SUPABASE_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY, + SUPABASE_SECRET_KEY: env.SECRET_KEY ?? env.SERVICE_ROLE_KEY, + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + NODE_ENV: 'development', + }, + }); + console.log( + 'seeded. next: pnpm local docs api (keep it running in a separate terminal)' + ); +} + +function cmdApi(flags: Map) { + const docs = docsPath(flags); + const docsApp = join(docs, 'apps', 'docs'); + const port = flags.get('port') ?? '3001'; + const env = stackEnv(); + const tsx = join( + docsApp, + 'node_modules', + '.bin', + onWindows ? 'tsx.cmd' : 'tsx' + ); + if (!existsSync(tsx)) + fail( + `tsx not installed in the docs app — run:\n corepack pnpm --dir ${docs} install --filter ./apps/docs...` + ); + const stub = join(__dirname, 'docs', 'sentry-stub-register.mjs'); + console.log( + `serving on http://127.0.0.1:${port}/docs/api/graphql — point evals at it with --content-api` + ); + // Runs with the DOCS app's tsx + tsconfig so the route's TS and its + // `react-server` condition resolve; the Sentry stub no-ops the route's + // telemetry (the real package crashes outside Next's instrumentation). + run( + tsx, + [ + '--conditions=react-server', + '--tsconfig', + 'tsconfig.json', + join(__dirname, 'docs', 'content-api-server.ts'), + ], + { + cwd: docsApp, + shim: true, + env: { + NODE_ENV: 'development', + PORT: port, + DOCS_ROUTE_PATH: join(docsApp, 'app', 'api', 'graphql', 'route.ts'), + NEXT_PUBLIC_SUPABASE_URL: env.API_URL, + NEXT_PUBLIC_SUPABASE_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY, + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + NODE_OPTIONS: `--import ${stub}${process.env.NODE_OPTIONS ? ` ${process.env.NODE_OPTIONS}` : ''}`, + }, + } + ); +} + +export async function main(argv: string[]) { + const [sub, ...rest] = argv; + const flags = new Map(); + const boolFlags = new Set(['yes']); + for (let i = 0; i < rest.length; i++) { + const a = rest[i]; + if (!a.startsWith('--')) continue; + const name = a.slice(2); + if (boolFlags.has(name)) flags.set(name, '1'); + else { + flags.set(name, rest[i + 1] ?? ''); + i++; + } + } + switch (sub) { + case 'up': + cmdUp(flags); + break; + case 'seed': + await cmdSeed(flags); + break; + case 'api': + cmdApi(flags); + break; + case 'down': + run('supabase', ['stop', '--workdir', OVERLAY]); + break; + default: + fail( + 'usage: pnpm local docs [--docs ] [--port N] [--yes]' + ); + } +} diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts new file mode 100755 index 00000000..3f7e704c --- /dev/null +++ b/apps/framework/scripts/local.ts @@ -0,0 +1,446 @@ +#!/usr/bin/env tsx +/** + * local.ts — local-dev runner. Run evals against YOUR inputs (edited skills + * tree, a local MCP build, a custom docs content API) with provenance + * receipts, and optionally compare against the latest published results on + * `origin/main`. + * + * pnpm local run [--experiment ] [--runs N] [--mcp ] [--content-api ] + * pnpm local compare [same flags] + * pnpm local experiments + * pnpm local docs [--docs ] (see local-docs.ts) + * + * Design notes: + * - Treatment-only: nothing here ever mutates a git tree, so concurrent + * sessions/worktrees cannot interfere and in-flight work is never at risk. + * - `compare` is a SCREEN, not causal proof: the published arm ran in the + * scheduled CI world (published MCP package, prod docs index, model state + * at refresh time). The receipt records the published result commit, its + * parent, and its age so the gap is explicit. + * - Explicit over magic: this does not build your MCP checkout or re-embed + * docs for you; it reports what world it measured. Build with + * `pnpm build` in your mcp checkout; serve docs with `pnpm local docs`. + */ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from 'node:fs'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; +import { + getExperimentDisplayMetadata, + type ExperimentConfig, +} from '@supabase-evals/core'; +import { main as docsMain } from './local-docs.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, '..', '..', '..'); +const OUT_DIR = join(ROOT, 'results-local'); +const PUBLISHED_FILES = [ + 'apps/web/src/data/regression-eval-results.json', + 'apps/web/src/data/eval-results.json', +]; +const DEFAULT_EXPERIMENT = 'claude-code-sonnet-5'; + +// ---------- tiny arg helpers (single-value flags; positionals collected) ---------- + +function parseArgs(argv: string[]) { + const flags = new Map(); + const bools = new Set(); + const positionals: string[] = []; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (!a.startsWith('--')) { + positionals.push(a); + continue; + } + const eq = a.indexOf('='); + if (eq !== -1) { + flags.set(a.slice(2, eq), a.slice(eq + 1)); + continue; + } + const next = argv[i + 1]; + if (next !== undefined && !next.startsWith('--')) { + flags.set(a.slice(2), next); + i++; + } else { + bools.add(a.slice(2)); + } + } + return { flags, bools, positionals }; +} + +function fail(msg: string): never { + console.error(msg); + process.exit(1); +} + +// ---------- git helpers (plain child_process; cross-platform) ---------- + +function git(args: string[], cwd: string = ROOT): string { + return execFileSync('git', args, { cwd, maxBuffer: 1 << 28 }) + .toString() + .trim(); +} + +function tryGit(args: string[], cwd: string = ROOT): string | undefined { + try { + return git(args, cwd); + } catch { + return undefined; + } +} + +// ---------- provenance receipts ---------- + +type Provenance = { + generatedAt: string; + host: { sha?: string; branch?: string; dirtyFiles: number }; + mcpOverride?: { path: string; sha?: string; dirtyFiles?: number }; + contentApiUrl?: string; + platform: string; +}; + +function collectProvenance(mcpPath?: string, contentApi?: string): Provenance { + const dirty = (cwd: string) => + (tryGit(['status', '--porcelain'], cwd) ?? '').split('\n').filter(Boolean) + .length; + const p: Provenance = { + generatedAt: new Date().toISOString(), + host: { + sha: tryGit(['rev-parse', 'HEAD']), + branch: tryGit(['rev-parse', '--abbrev-ref', 'HEAD']), + dirtyFiles: dirty(ROOT), + }, + platform: `${process.platform}/${process.arch} node ${process.version}`, + }; + if (mcpPath) { + const inRepo = tryGit(['rev-parse', '--show-toplevel'], mcpPath); + p.mcpOverride = { + path: mcpPath, + sha: inRepo ? tryGit(['rev-parse', 'HEAD'], mcpPath) : undefined, + dirtyFiles: inRepo + ? (tryGit(['status', '--porcelain'], inRepo) ?? '') + .split('\n') + .filter(Boolean).length + : undefined, + }; + } + if (contentApi) p.contentApiUrl = contentApi; + return p; +} + +// ---------- published baselines (compare mode) ---------- + +type PublishedRow = { + experiment: string; + eval: string; + passed?: boolean; + attempts?: number; + checks?: Array<{ name: string; passed: boolean }>; + docs?: { calls?: unknown[] }; + [k: string]: unknown; +}; + +type Baseline = { + row: PublishedRow; + file: string; + commit: string; + parent: string; + committedAt: string; +}; + +function resolveBaselines( + evalIds: string[], + experiment: string, + fetch: boolean +): Map { + if (fetch) { + try { + git(['fetch', '-q', 'origin', 'main']); + } catch { + console.error( + 'warning: could not fetch origin/main — comparing against the local ref, which may be stale' + ); + } + } + const best = new Map(); + const seen = new Map>(); + for (const file of PUBLISHED_FILES) { + let rows: PublishedRow[]; + try { + rows = JSON.parse(git(['show', `origin/main:${file}`])); + } catch { + continue; + } + const [commit, parent, committedAt] = git([ + 'log', + 'origin/main', + '-1', + '--format=%H %P %cI', + '--', + file, + ]).split(' '); + for (const row of rows) { + if (!evalIds.includes(row.eval)) continue; + if (!seen.has(row.eval)) seen.set(row.eval, new Set()); + seen.get(row.eval)?.add(row.experiment); + if (row.experiment !== experiment) continue; + const cur = best.get(row.eval); + if (!cur || new Date(committedAt) > new Date(cur.committedAt)) + best.set(row.eval, { row, file, commit, parent, committedAt }); + } + } + const missing = evalIds.filter((e) => !best.has(e)); + if (missing.length) { + for (const e of missing) { + const alts = [...(seen.get(e) ?? [])]; + console.error( + alts.length + ? `no published ${experiment} result for ${e} on origin/main (published experiments: ${alts.join(', ')})` + : `no published result for ${e} on origin/main at all — use \`pnpm local run\` (no baseline needed)` + ); + } + process.exit(1); + } + return best; +} + +// ---------- eval validation (fail before spending) ---------- + +function validateEvals(evalIds: string[]) { + for (const id of evalIds) { + const promptPath = join(ROOT, 'evals', id, 'PROMPT.md'); + if (!existsSync(promptPath)) + fail(`no eval at evals/${id} (PROMPT.md missing)`); + try { + parseEvalMarkdown( + readFileSync(promptPath, 'utf8'), + `evals/${id}/PROMPT.md` + ); + } catch (err) { + fail( + `eval metadata invalid — fix evals/${id}/PROMPT.md before spending on runs\n${err instanceof Error ? err.message : String(err)}` + ); + } + } +} + +function validateExperiment(experiment: string) { + if (!existsSync(join(ROOT, 'experiments', `${experiment}.ts`))) { + const available = readdirSync(join(ROOT, 'experiments')) + .filter((f) => f.endsWith('.ts')) + .map((f) => f.replace(/\.ts$/, '')); + fail( + `unknown experiment: ${experiment}\navailable: ${available.join(', ')}\n(or add experiments/${experiment}.ts — see any existing file for the shape)` + ); + } +} + +// ---------- treatment run ---------- + +function runEval( + evalId: string, + experiment: string, + runs: number, + env: Record +): string { + const res = spawnSync( + process.execPath, + [ + '--import', + 'tsx/esm', + join(__dirname, '..', 'harness', 'run-eval.ts'), + '--eval', + evalId, + '--experiment', + experiment, + '--runs', + String(runs), + ], + { + stdio: 'inherit', + cwd: join(__dirname, '..'), + env: { ...process.env, ...env }, + } + ); + if (res.status !== 0) fail(`eval run failed: ${evalId} (exit ${res.status})`); + const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`); + if (!existsSync(resultPath)) + fail( + `no result at results/${experiment}/${evalId}.json — check the eval/experiment ids` + ); + return resultPath; +} + +// ---------- reporting ---------- + +function reportRow( + label: string, + r: PublishedRow | undefined, + extra: string +): string { + const checks = r?.checks ?? []; + const checksSummary = `${checks.filter((x) => x.passed).length}/${checks.length}`; + const docsCalls = r?.docs?.calls?.length ?? 0; + return `${label.padEnd(10)} passed=${String(r?.passed).padEnd(5)} checks=${checksSummary.padEnd(6)} docs.calls=${String(docsCalls).padEnd(3)} ${extra}`; +} + +// ---------- subcommands ---------- + +async function cmdExperiments() { + const published = new Set(); + for (const file of PUBLISHED_FILES) { + try { + for (const row of JSON.parse( + git(['show', `origin/main:${file}`]) + ) as PublishedRow[]) + published.add(row.experiment); + } catch { + /* offline or file missing: published column degrades to '-' */ + } + } + console.log( + `${'EXPERIMENT'.padEnd(36)} ${'AGENT'.padEnd(12)} ${'MODEL'.padEnd(22)} ${'EFFORT'.padEnd(8)} PUBLISHED` + ); + for (const f of readdirSync(join(ROOT, 'experiments')) + .filter((f) => f.endsWith('.ts')) + .sort()) { + const name = f.replace(/\.ts$/, ''); + // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments) + const mod = await import(pathToFileURL(join(ROOT, 'experiments', f)).href); + const display = getExperimentDisplayMetadata( + mod.default as ExperimentConfig + ); + console.log( + `${name.padEnd(36)} ${(display.agent ?? '?').padEnd(12)} ${(display.modelId ?? '?').padEnd(22)} ${(display.reasoningEffort ?? '-').padEnd(8)} ${published.has(name) ? 'yes (compare)' : '-'}` + ); + } +} + +function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) { + const { flags, positionals } = parseArgs(argv); + const evalIds = positionals; + if (!evalIds.length) + fail( + `usage: pnpm local ${mode} [...] [--experiment ] [--runs N] [--mcp ] [--content-api ]` + ); + const experiment = flags.get('experiment') ?? DEFAULT_EXPERIMENT; + validateExperiment(experiment); + + const baselines = + mode === 'compare' + ? resolveBaselines(evalIds, experiment, !process.env.LOCAL_NO_FETCH) + : new Map(); + + validateEvals(evalIds); + + const env: Record = {}; + let mcpPath = flags.get('mcp'); + if (mcpPath) { + mcpPath = isAbsolute(mcpPath) ? mcpPath : resolve(process.cwd(), mcpPath); + if (!existsSync(mcpPath)) fail(`--mcp path does not exist: ${mcpPath}`); + env.SUPABASE_MCP_SERVER_PATH = mcpPath; + } + const contentApi = flags.get('content-api'); + if (contentApi) env.SUPABASE_CONTENT_API_URL = contentApi; + + mkdirSync(OUT_DIR, { recursive: true }); + let exitCode = 0; + for (const id of evalIds) { + const runs = Number( + flags.get('runs') ?? baselines.get(id)?.row.attempts ?? 1 + ); + console.log( + `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}${contentApi ? ', content-api override' : ''}) ==` + ); + const resultPath = process.env.LOCAL_EVAL_CMD + ? fakeRun(id, experiment) + : runEval(id, experiment, runs, env); + + const result = JSON.parse(readFileSync(resultPath, 'utf8')) as PublishedRow; + const receipt = { + ...result, + provenance: collectProvenance(mcpPath, contentApi), + }; + const treatmentPath = join(OUT_DIR, `${id}.treatment.json`); + writeFileSync(treatmentPath, `${JSON.stringify(receipt, null, 1)}\n`); + + console.log(`\n=== local ${mode}: ${id} (${experiment}) ===`); + const b = baselines.get(id); + if (b) { + writeFileSync( + join(OUT_DIR, `${id}.published.json`), + `${JSON.stringify({ ...b.row, publishedProvenance: { file: b.file, commit: b.commit, parent: b.parent, committedAt: b.committedAt } }, null, 1)}\n` + ); + const ageDays = Math.round( + (Date.now() - new Date(b.committedAt).getTime()) / 86_400_000 + ); + console.log( + reportRow( + 'published', + b.row, + `main@${b.commit.slice(0, 7)} ${b.committedAt.slice(0, 10)} (${ageDays}d old, attempts ${b.row.attempts})` + ) + ); + console.log(reportRow('treatment', result, 'your world')); + const d = (result.passed ? 1 : 0) - (b.row.passed ? 1 : 0); + console.log( + d > 0 + ? '-> IMPROVED vs published (FAIL->PASS)' + : d < 0 + ? '-> REGRESSED vs published (PASS->FAIL)' + : '-> no pass/fail change (compare checks / docs.calls)' + ); + if (d < 0) exitCode = 1; + console.log( + 'screen only: the published arm ran in the scheduled CI world — a flip is a signal, not causal proof' + ); + console.log(`saved: results-local/${id}.{published,treatment}.json`); + } else { + console.log(reportRow('treatment', result, 'your world')); + console.log(`saved: results-local/${id}.treatment.json`); + } + } + process.exit(exitCode); +} + +// test hook: LOCAL_EVAL_CMD writes the result file itself (no model spend) +function fakeRun(evalId: string, experiment: string): string { + const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`); + mkdirSync(dirname(resultPath), { recursive: true }); + const res = spawnSync(process.env.LOCAL_EVAL_CMD as string, { + shell: true, + stdio: 'inherit', + env: { ...process.env, RES: resultPath, EVAL: evalId }, + }); + if (res.status !== 0) fail(`LOCAL_EVAL_CMD failed for ${evalId}`); + return resultPath; +} + +// ---------- entry ---------- + +const [command, ...rest] = process.argv.slice(2); +switch (command) { + case 'run': + case 'compare': + cmdRunOrCompare(command, rest); + break; + case 'experiments': + await cmdExperiments(); + break; + case 'docs': + await docsMain(rest); + break; + default: + fail(`usage: pnpm local ... + run run eval(s) in your world (skills tree as-is; --mcp / --content-api overrides) + compare run + diff against the latest published result on origin/main + experiments list experiments (agent, model, effort, published-baseline availability) + docs --docs local docs content API`); +} diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts new file mode 100644 index 00000000..4dc549a4 --- /dev/null +++ b/apps/framework/scripts/smoke-local.ts @@ -0,0 +1,150 @@ +/** + * Zero-cost smoke test for the local-dev runner (scripts/local.ts). + * + * Fakes the eval run via LOCAL_EVAL_CMD (no model spend, no docker) and + * reads REAL published baselines from origin/main (no fetch: LOCAL_NO_FETCH). + * + * pnpm --filter @supabase-evals/framework test:local + */ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, readFileSync, rmSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(__dirname, '..', '..', '..'); +const EXPERIMENT = 'claude-code-sonnet-5'; + +// a published, currently-existing eval id — resolved dynamically so the test +// doesn't rot when the published set changes +const published = JSON.parse( + execFileSync( + 'git', + ['show', 'origin/main:apps/web/src/data/regression-eval-results.json'], + { cwd: ROOT, maxBuffer: 1 << 28 } + ).toString() +) as Array<{ experiment: string; eval: string }>; +const EVAL = published.find( + (r) => r.experiment === EXPERIMENT && existsSync(join(ROOT, 'evals', r.eval)) +)?.eval; +assert.ok(EVAL, 'no published eval with a local evals/ dir found'); + +// LOCAL_EVAL_CMD contract: write a result JSON to $RES for eval $EVAL +const FAKE = + process.platform === 'win32' + ? `node -e "require('fs').mkdirSync(require('path').dirname(process.env.RES),{recursive:true});require('fs').writeFileSync(process.env.RES,JSON.stringify({eval:process.env.EVAL,experiment:'${EXPERIMENT}',passed:true,checks:[{name:'x',passed:true}]}))"` + : `node -e 'require("fs").mkdirSync(require("path").dirname(process.env.RES),{recursive:true});require("fs").writeFileSync(process.env.RES,JSON.stringify({eval:process.env.EVAL,experiment:"${EXPERIMENT}",passed:true,checks:[{name:"x",passed:true}]}))'`; + +function local(args: string[], env: Record = {}) { + const res = spawnSync( + process.execPath, + ['--import', 'tsx/esm', join(__dirname, 'local.ts'), ...args], + { + cwd: join(__dirname, '..'), + encoding: 'utf8', + env: { + ...process.env, + LOCAL_NO_FETCH: '1', + LOCAL_EVAL_CMD: FAKE, + FORCE_COLOR: '0', + ...env, + }, + } + ); + return { out: `${res.stdout}\n${res.stderr}`, status: res.status }; +} + +let passed = 0; +function ck(name: string, fn: () => void) { + try { + fn(); + passed++; + } catch (err) { + console.error(`FAIL: ${name}`); + throw err; + } +} + +// --- refusals happen pre-spend, with actionable messages --- +{ + const r = local(['compare', 'no-such-eval-xyz']); + ck('unknown eval refused', () => { + assert.equal(r.status, 1); + assert.match(r.out, /no published result for no-such-eval-xyz/); + }); +} +{ + const r = local(['compare', EVAL, '--experiment', 'bogus-model']); + ck('unknown experiment refused with the available list', () => { + assert.equal(r.status, 1); + assert.match(r.out, /unknown experiment: bogus-model/); + assert.match(r.out, /claude-code-sonnet-5/); + }); +} +{ + const r = local(['run', 'not-an-eval-dir']); + ck('missing eval dir refused', () => { + assert.equal(r.status, 1); + assert.match(r.out, /no eval at evals\/not-an-eval-dir/); + }); +} + +// --- compare: delta table + receipts with published provenance --- +{ + const r = local(['compare', EVAL]); + ck('compare prints both rows and the screen caveat', () => { + assert.equal(r.status, 0); + assert.match(r.out, new RegExp(`=== local compare: ${EVAL}`)); + assert.match(r.out, /published .*main@[0-9a-f]{7}/); + assert.match(r.out, /treatment .*your world/); + assert.match(r.out, /screen only:/); + }); + ck('published receipt carries commit provenance', () => { + const receipt = JSON.parse( + readFileSync( + join(ROOT, 'results-local', `${EVAL}.published.json`), + 'utf8' + ) + ); + assert.match(receipt.publishedProvenance.commit, /^[0-9a-f]{40}$/); + assert.match(receipt.publishedProvenance.parent, /^[0-9a-f]{40}$/); + }); + ck('treatment receipt carries host provenance', () => { + const receipt = JSON.parse( + readFileSync( + join(ROOT, 'results-local', `${EVAL}.treatment.json`), + 'utf8' + ) + ); + assert.match(receipt.provenance.host.sha, /^[0-9a-f]{40}$/); + assert.equal(typeof receipt.provenance.host.dirtyFiles, 'number'); + }); +} + +// --- run: no baseline required (custom evals), receipt only --- +{ + const r = local(['run', EVAL]); + ck('run works without published baseline machinery', () => { + assert.equal(r.status, 0); + assert.match(r.out, new RegExp(`=== local run: ${EVAL}`)); + assert.doesNotMatch(r.out, /published /); + assert.match(r.out, /saved: results-local\//); + }); +} + +// --- mcp override path validation --- +{ + const r = local(['run', EVAL, '--mcp', '/definitely/not/a/path']); + ck('bad --mcp path refused pre-spend', () => { + assert.equal(r.status, 1); + assert.match(r.out, /--mcp path does not exist/); + }); +} + +// cleanup +rmSync(join(ROOT, 'results-local', `${EVAL}.published.json`), { force: true }); +rmSync(join(ROOT, 'results-local', `${EVAL}.treatment.json`), { force: true }); +rmSync(join(ROOT, 'results', EXPERIMENT, `${EVAL}.json`), { force: true }); + +console.log(`smoke-local: ${passed} checks passed`); diff --git a/package.json b/package.json index 08a0ef27..9394b08c 100644 --- a/package.json +++ b/package.json @@ -1,40 +1,44 @@ { - "name": "supabase-evals", - "private": true, - "type": "module", - "workspaces": ["apps/*", "packages/*"], - "scripts": { - "check": "pnpm --filter @supabase-evals/framework check && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test", - "eval": "pnpm --filter @supabase-evals/framework eval", - "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry", - "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke", - "eval:force": "pnpm --filter @supabase-evals/framework eval:force", - "test:framework": "pnpm --filter @supabase-evals/framework test:framework", - "export-results": "pnpm --filter @supabase-evals/framework export-results", - "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", - "web": "pnpm --filter @supabase-evals/web dev", - "web:build": "pnpm --filter @supabase-evals/web build", - "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp", - "demo:executor": "pnpm --filter @supabase-evals/framework demo:executor", - "format": "biome check --write . && pnpm --filter @supabase-evals/web format", - "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check" - }, - "dependencies": { - "@ai-sdk/anthropic": "catalog:", - "@ai-sdk/openai": "catalog:", - "common-tags": "^1.8.2" - }, - "devDependencies": { - "@biomejs/biome": "1.9.4", - "@supabase-evals/core": "workspace:*", - "@supabase-evals/sandbox": "workspace:*", - "@types/common-tags": "^1.8.4", - "@types/node": "catalog:", - "typescript": "catalog:" - }, - "engines": { - "node": ">=22", - "pnpm": "10.24" - }, - "packageManager": "pnpm@10.24.0" + "name": "supabase-evals", + "private": true, + "type": "module", + "workspaces": [ + "apps/*", + "packages/*" + ], + "scripts": { + "check": "pnpm --filter @supabase-evals/framework check && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test", + "eval": "pnpm --filter @supabase-evals/framework eval", + "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry", + "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke", + "eval:force": "pnpm --filter @supabase-evals/framework eval:force", + "test:framework": "pnpm --filter @supabase-evals/framework test:framework", + "export-results": "pnpm --filter @supabase-evals/framework export-results", + "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", + "web": "pnpm --filter @supabase-evals/web dev", + "web:build": "pnpm --filter @supabase-evals/web build", + "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp", + "demo:executor": "pnpm --filter @supabase-evals/framework demo:executor", + "format": "biome check --write . && pnpm --filter @supabase-evals/web format", + "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check", + "local": "pnpm --filter @supabase-evals/framework local" + }, + "dependencies": { + "@ai-sdk/anthropic": "catalog:", + "@ai-sdk/openai": "catalog:", + "common-tags": "^1.8.2" + }, + "devDependencies": { + "@biomejs/biome": "1.9.4", + "@supabase-evals/core": "workspace:*", + "@supabase-evals/sandbox": "workspace:*", + "@types/common-tags": "^1.8.4", + "@types/node": "catalog:", + "typescript": "catalog:" + }, + "engines": { + "node": ">=22", + "pnpm": "10.24" + }, + "packageManager": "pnpm@10.24.0" } From 346db9cf3e25e4777966ade355f11c94c1537ba8 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Mon, 27 Jul 2026 09:37:57 +0200 Subject: [PATCH 03/24] docs: local development loop section --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 7662b01a..ff190ab0 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,42 @@ Start the web app development server: pnpm web ``` +## Local development loop (`pnpm local`) + +Testing a change to an agent input — a skill, a local build of +[`mcp-server-supabase`](https://github.com/supabase/mcp), or an edited docs +page — against the evals, without touching git state: + +```bash +pnpm local run [--experiment ] [--mcp ] [--content-api ] +pnpm local compare [same flags] # + diff vs the latest published result on main +pnpm local experiments # list experiments + published-baseline availability +``` + +- **Skills**: edit the skills tree in this repo and just `run` — the harness + reads it as-is. +- **MCP**: clone + build the mcp repo anywhere, then `--mcp ` + (sets `SUPABASE_MCP_SERVER_PATH`, so `search_docs` and friends run your build). +- **Docs**: serve a local docs content API from your own supabase/supabase + checkout, then point runs at it: + + ```bash + pnpm local docs up --docs + pnpm local docs seed # full embed via the docs app's pipeline (~$0.12 OpenAI; asks first) + pnpm local docs api # keep running in a separate terminal + pnpm local run --content-api http://127.0.0.1:3001/docs/api/graphql + ``` + +Every run writes a provenance receipt to `results-local/` (host SHA + dirty +state, override paths and their git state). `compare` records the published +arm's result commit, parent, and age — and a pass/fail flip against published +is a **screen**, not causal proof: the published run happened in the scheduled +CI world (published mcp package, prod docs index, model state at refresh time). + +Keys go in `.env` at the repo root (`ANTHROPIC_API_KEY`, and `OPENAI_API_KEY` +for the docs loop). Zero-cost self-test: `pnpm --filter +@supabase-evals/framework test:local`. + ## Eval Shape Every eval contains: From 2cc2af4c131318cd1428aac4ccba94c387dee468 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Mon, 27 Jul 2026 09:56:23 +0200 Subject: [PATCH 04/24] docs: design brief for the local-workflows single-page presentation Self-contained brief for a design agent: content (final copy for all three workflows), page structure, comparison table, visual direction, and constraints for a single-file HTML page presenting the pnpm local skills/mcp/docs workflows. --- docs/local-workflows-design-brief.md | 180 +++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 docs/local-workflows-design-brief.md diff --git a/docs/local-workflows-design-brief.md b/docs/local-workflows-design-brief.md new file mode 100644 index 00000000..2743bf7c --- /dev/null +++ b/docs/local-workflows-design-brief.md @@ -0,0 +1,180 @@ +# Design brief: "Test your change against the evals" — single-page HTML + +Audience for this document: a design agent implementing a beautiful, +self-contained, single-page HTML presentation. Everything needed (copy, +structure, data, constraints) is in this brief; no repo access required. + +## Purpose & audience + +One page that teaches a Supabase engineer the three local eval workflows in +under two minutes of scanning: + +> I changed an agent input — a **skill**, the **MCP server**, or a **docs +> page**. How do I verify the change improved an eval, or at least didn't +> regress one? + +Viewers are engineers. They want the mental model, the exact commands, and +the honest limits — in that order. The page is presentation-first (screen +share in a team meeting, then linked in Slack), so it must read well both +projected and self-served. + +## The one mental model (hero concept) + +The page hangs on a single idea — **inputs are explicit overrides**: + +- An eval run is `agent + inputs -> score`. +- The three inputs come from *your* machine, not managed clones: the skills + tree lives in the evals repo; the MCP server is your own checkout passed + via `--mcp`; docs are served from your own supabase/supabase checkout via + `--content-api`. +- One command family drives everything: `pnpm local …`. +- Every run leaves a **provenance receipt** (what world was measured), and + every comparison against published results is a **screen, not causal + proof** (the published arm ran in CI's world at refresh time). + +Suggested hero: the equation/flow rendered visually — + +``` + skills tree ─┐ + --mcp ─┼─> pnpm local run/compare ─> verdict + receipt + --content-api ┘ +``` + +## Page structure (top to bottom) + +1. **Hero**: title + the mental model + the flow graphic. + - Title suggestion: "Test your change against the evals" + - Subtitle: "Three inputs, one command family, honest verdicts." +2. **Decision strip** ("What did you change?") — three buttons/cards that + anchor-link to the workflow sections: Skill · MCP server · Docs page. +3. **Three workflow sections** (content below). Consistent internal layout: + speed/cost badges → 3-4 step commands → "what the verdict means" note. +4. **Shared semantics band**: receipts, pre-spend gates, screen-vs-proof. +5. **Comparison table** (the three loops side by side). +6. **Footer**: links (placeholders): PR #128, README section "Local + development loop", repo. + +## Section content (copy is final; do not rewrite technical strings) + +### Workflow 1 — Skills · badges: `fastest` `$0 extra` + +The skills tree lives in the evals repo itself. Edit and run; the harness +reads it as-is. + +```bash +vim skills/supabase/... # 1. edit the skill +pnpm local compare # 2. run + diff vs the published result +``` + +Note: iteration cost is model spend only. No build step, no services. + +### Workflow 2 — MCP server · badges: `rebuild in seconds` `$0 extra` + +Your own checkout, built locally, passed explicitly. + +```bash +git clone https://github.com/supabase/mcp ~/dev/mcp # once +cd ~/dev/mcp && pnpm install && pnpm build # once + +vim ~/dev/mcp/packages/mcp-server-supabase/src/... # 1. edit +pnpm build # 2. rebuild (seconds) +pnpm local compare --mcp ~/dev/mcp # 3. run + diff +``` + +Note (render as a callout): judge MCP changes by **tool-call activation**, +not pass/fail alone — an eval can pass without ever calling the tool you +changed. The receipt records which tools were called. + +### Workflow 3 — Docs page · badges: `needs Docker` `~$0.12 per re-embed` + +Docs are served from your own supabase/supabase checkout through a local +content API; the eval's `search_docs` reads your index. + +```bash +git clone https://github.com/supabase/supabase ~/dev/supabase # once +pnpm local docs up --docs ~/dev/supabase # once per session +pnpm local docs seed # embed (~$0.12, asks first) +pnpm local docs api # separate terminal, keep running + +vim ~/dev/supabase/apps/docs/content/guides/...mdx # 1. edit +pnpm local docs seed --yes # 2. re-embed (~$0.12) +pnpm local compare \ + --content-api http://127.0.0.1:3001/docs/api/graphql # 3. run + diff +``` + +Note (render as a callout): pick an eval that can *see* the docs — a +tools-mode eval whose answer lives in the edited page. Incremental +re-embeds (cents instead of $0.12) arrive once the upstream pipeline fixes +land. + +### Shared semantics band (applies to all three) + +- **Verdict**: `compare` prints published vs treatment (pass/fail, checks, + docs calls) and one line: IMPROVED · REGRESSED · no change. Nonzero exit + on regression, so it works as a gate. +- **Receipts**: every run writes `results-local/.treatment.json` — + host SHA + dirty state, override paths and their git state; `compare` + adds the published arm's result commit, parent, and age. +- **Screen, not proof** (give this visual weight): the published arm ran in + the scheduled CI world (published MCP package, prod docs index, model + state at refresh time). One run is n=1. Before claiming a number moved: + `--runs 3`, read check-level results, not just pass/fail. +- **Pre-spend gates**: invalid eval metadata, unknown experiment, or a bad + `--mcp` path refuse *before* any model call. +- **No baseline? No problem**: custom evals (not in the published set) use + `pnpm local run` — same receipts, no comparison row. + +### Comparison table + +| | Skills | MCP server | Docs page | +|---|---|---|---| +| Where you edit | `skills/` in the evals repo | your mcp checkout | your supabase/supabase checkout | +| Sync step | none | `pnpm build` (~seconds) | `pnpm local docs seed` (~$0.12) | +| Services needed | none | none | Docker + supabase CLI + docs api terminal | +| Extra flag | — | `--mcp ` | `--content-api ` | +| Iteration cost | model runs only | model runs only | model runs + ~$0.12 embed | +| Judge by | checks | tool-call activation + checks | docs.calls + checks | + +## Visual direction + +- **Supabase brand**: dark theme (near-black background, e.g. #0F0F0F / + #1C1C1C surfaces), Supabase green `#3ECF8E` as THE accent (verdicts, + badges, active states), off-white text. Generous whitespace; feels like + supabase.com, not a wiki. +- Typography: a clean geometric sans for headings (Circular-adjacent; system + fallback fine), high-quality monospace for commands (JetBrains Mono / + ui-monospace). +- Code blocks are first-class citizens: syntax-tinted, copy-to-clipboard + button, step numbers rendered in the gutter (the `# 1.` comments above may + become styled step markers). +- Each workflow gets an icon and an accent tint within the green family; + badges are small pills (speed/cost) at the section top. +- The "screen, not proof" message deserves a distinct visual treatment — an + amber/neutral callout, NOT an error style. It's honesty, not a warning. +- A rendered verdict example adds credibility — mock a small terminal card: + + ``` + === local compare: docs-rls-discovery (claude-code-sonnet-5) === + published passed=false checks=0/1 docs.calls=2 main@ccdacd9 2026-07-25 (0d old) + treatment passed=true checks=1/1 docs.calls=3 your world + -> IMPROVED vs published (FAIL->PASS) + ``` + + (Illustrative output; keep the shape, values may be stylized.) + +## Constraints + +- **Single self-contained `.html` file**: inline CSS + minimal inline JS + (copy buttons, anchor scrolling). No build step, no external JS. System + fonts or one font via CDN link at most; page must degrade gracefully + offline. +- Responsive: presentable projected at 1920w, readable at 375w. +- No screenshots of real dashboards; everything drawn/styled. +- Accessible: real semantic headings, code in `
`, contrast AA.
+- Keep total page weight small (<200KB without fonts).
+
+## Out of scope
+
+- No interactive terminal emulation, no animation beyond subtle hover/entry.
+- Do not invent additional workflows, flags, or costs beyond this brief.
+- Command strings, flag names, paths, and prices are exact — do not edit.

From 17e5da4750c0fab3680941abd93ea7e4effb1318 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 11:13:55 +0200
Subject: [PATCH 05/24] =?UTF-8?q?docs:=20AGENTS.md=20=E2=80=94=20agent=20i?=
 =?UTF-8?q?nstructions=20for=20the=20local=20eval=20workflows?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Operational rules for coding agents: how to verify skills/mcp/docs changes
with pnpm local, interpretation rules (screen not proof, --runs 3 before
claims, tool-call activation for mcp, tools-mode evals for docs), spend
rules (docs seed ~$0.12, confirm before paid steps, don't bypass pre-spend
gates), and repo conventions. CLAUDE.md points at it.
---
 AGENTS.md | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 CLAUDE.md |  1 +
 2 files changed, 79 insertions(+)
 create mode 100644 AGENTS.md
 create mode 100644 CLAUDE.md

diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 00000000..15ba1cb5
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,78 @@
+# AGENTS.md — supabase/evals
+
+Instructions for coding agents working in this repo. Humans: start with
+[README.md](README.md) and [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## What this repo is
+
+Evals for Supabase AI agents. An eval run is `agent + inputs -> score`. The
+three inputs a change usually targets: the **skills** tree (in this repo),
+the **MCP server** (external checkout), and **docs** content (external
+supabase/supabase checkout).
+
+## Verifying a change against the evals (`pnpm local`)
+
+Use the local runner for all "did my change help / did it regress?" work.
+It never mutates git state, so it is safe alongside in-flight work.
+
+```bash
+pnpm local run      [--experiment ] [--runs N] [--mcp ] [--content-api ]
+pnpm local compare  [same flags]   # + diff vs latest published result on origin/main
+pnpm local experiments                         # experiments + which have published baselines
+pnpm local docs  --docs 
+```
+
+Per input:
+
+- **Skill edited** (in `skills/`): no sync step — `pnpm local compare `.
+- **MCP server edited** (external checkout): `pnpm build` in that checkout,
+  then `pnpm local compare  --mcp `.
+- **Docs page edited** (external supabase/supabase checkout):
+  `pnpm local docs seed --yes` to re-embed (**~$0.12 OpenAI — see spend
+  rules**), keep `pnpm local docs api` running in a separate terminal, then
+  `pnpm local compare  --content-api http://127.0.0.1:3001/docs/api/graphql`.
+
+Receipts land in `results-local/` (git-ignored): treatment provenance (host
+SHA + dirty state, override git state) and, for `compare`, the published
+arm's result commit + parent + age.
+
+## Interpreting results — rules, not suggestions
+
+- **`compare` is a screen, not causal proof.** The published arm ran in the
+  scheduled CI world (published MCP package, prod docs index, model state at
+  refresh time). Never report a flip as caused by the edit; report it as a
+  signal consistent with the edit.
+- **Single runs are noisy.** Before claiming improvement or regression, run
+  `--runs 3` and read check-level results, not just pass/fail.
+- **MCP changes: judge by tool-call activation.** An eval can pass without
+  ever calling the tool you changed. Confirm the changed tool was actually
+  exercised (the result JSON records tool calls) before concluding anything.
+- **Docs changes: the eval must be able to see the docs.** Use a tools-mode
+  (`interface: mcp`) eval whose answer lives in the edited page and is
+  reached via `search_docs`. CLI-scaffold evals can pass regardless of docs.
+- **No published baseline?** Use `pnpm local run` (custom evals included).
+  For a before/after, run once before the edit and once after.
+
+## Spend rules
+
+- Eval runs cost model tokens; `pnpm local docs seed` costs **~$0.12 OpenAI
+  per invocation**. State the cost and get user confirmation before
+  running paid steps the user did not explicitly request.
+- The runner refuses pre-spend on invalid eval metadata, unknown
+  experiments, and bad `--mcp` paths — do not work around these gates.
+- Zero-cost checks: `pnpm --filter @supabase-evals/framework test:local`
+  (runner self-test), `pnpm local experiments`, `pnpm eval:dry`.
+
+## Conventions
+
+- Keys live in `.env` at the repo root (`ANTHROPIC_API_KEY`; plus
+  `OPENAI_API_KEY` for the docs loop). Never hardcode or echo key values.
+- Model/agent selection = experiment id. To test an unlisted model, add a
+  small `experiments/.ts` (copy an existing file's shape) rather than
+  editing a published experiment in place.
+- `results/`, `results-local/`, and `.local-docs/` are outputs — never
+  commit their contents.
+- Verify with `pnpm check` (typecheck + core/sandbox tests) and
+  `pnpm format:check` (biome) before pushing.
+- New evals: follow [CONTRIBUTING.md](CONTRIBUTING.md) (suite choice,
+  `motivation:` frontmatter, scorer shape).
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 00000000..5514a3e2
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1 @@
+See [AGENTS.md](AGENTS.md) for agent instructions in this repo.

From 6d3fa48d79b8a6832347c35cd990db5c6043e23a Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 11:15:15 +0200
Subject: [PATCH 06/24] chore: biome-format package.json edits

---
 apps/framework/package.json | 84 ++++++++++++++++++-------------------
 package.json                | 81 +++++++++++++++++------------------
 2 files changed, 81 insertions(+), 84 deletions(-)

diff --git a/apps/framework/package.json b/apps/framework/package.json
index 92fbe5b8..b047cbf3 100644
--- a/apps/framework/package.json
+++ b/apps/framework/package.json
@@ -1,44 +1,44 @@
 {
-	"name": "@supabase-evals/framework",
-	"private": true,
-	"version": "0.0.1",
-	"type": "module",
-	"scripts": {
-		"check": "pnpm typecheck && pnpm test:framework",
-		"eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
-		"eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
-		"eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
-		"typecheck": "tsc --noEmit",
-		"test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
-		"export-results": "node --import tsx/esm scripts/export-results.ts",
-		"demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
-		"demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts",
-		"local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/local.ts",
-		"test:local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-local.ts"
-	},
-	"dependencies": {
-		"@ai-sdk/anthropic": "catalog:",
-		"@ai-sdk/mcp": "catalog:",
-		"@ai-sdk/openai": "catalog:",
-		"@supabase-evals/platform-lite": "workspace:*",
-		"@electric-sql/pglite": "catalog:",
-		"@supabase-evals/core": "workspace:*",
-		"@supabase-evals/sandbox": "workspace:*",
-		"@supabase/supabase-js": "catalog:",
-		"@testing-library/jest-dom": "^6.9.1",
-		"@testing-library/react": "^16.3.2",
-		"@vitejs/plugin-react": "catalog:",
-		"ai": "catalog:",
-		"happy-dom": "^20.9.0",
-		"@supabase/lite": "catalog:",
-		"react": "^19.2.5",
-		"react-dom": "^19.2.5",
-		"vite": "catalog:",
-		"vitest": "catalog:"
-	},
-	"devDependencies": {
-		"@types/node": "catalog:",
-		"tsx": "^4.19.0",
-		"typescript": "catalog:"
-	}
+  "name": "@supabase-evals/framework",
+  "private": true,
+  "version": "0.0.1",
+  "type": "module",
+  "scripts": {
+    "check": "pnpm typecheck && pnpm test:framework",
+    "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
+    "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
+    "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
+    "typecheck": "tsc --noEmit",
+    "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
+    "export-results": "node --import tsx/esm scripts/export-results.ts",
+    "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
+    "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts",
+    "local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/local.ts",
+    "test:local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-local.ts"
+  },
+  "dependencies": {
+    "@ai-sdk/anthropic": "catalog:",
+    "@ai-sdk/mcp": "catalog:",
+    "@ai-sdk/openai": "catalog:",
+    "@supabase-evals/platform-lite": "workspace:*",
+    "@electric-sql/pglite": "catalog:",
+    "@supabase-evals/core": "workspace:*",
+    "@supabase-evals/sandbox": "workspace:*",
+    "@supabase/supabase-js": "catalog:",
+    "@testing-library/jest-dom": "^6.9.1",
+    "@testing-library/react": "^16.3.2",
+    "@vitejs/plugin-react": "catalog:",
+    "ai": "catalog:",
+    "happy-dom": "^20.9.0",
+    "@supabase/lite": "catalog:",
+    "react": "^19.2.5",
+    "react-dom": "^19.2.5",
+    "vite": "catalog:",
+    "vitest": "catalog:"
+  },
+  "devDependencies": {
+    "@types/node": "catalog:",
+    "tsx": "^4.19.0",
+    "typescript": "catalog:"
+  }
 }
diff --git a/package.json b/package.json
index 9394b08c..786bb7c3 100644
--- a/package.json
+++ b/package.json
@@ -1,44 +1,41 @@
 {
-	"name": "supabase-evals",
-	"private": true,
-	"type": "module",
-	"workspaces": [
-		"apps/*",
-		"packages/*"
-	],
-	"scripts": {
-		"check": "pnpm --filter @supabase-evals/framework check && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test",
-		"eval": "pnpm --filter @supabase-evals/framework eval",
-		"eval:dry": "pnpm --filter @supabase-evals/framework eval:dry",
-		"eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke",
-		"eval:force": "pnpm --filter @supabase-evals/framework eval:force",
-		"test:framework": "pnpm --filter @supabase-evals/framework test:framework",
-		"export-results": "pnpm --filter @supabase-evals/framework export-results",
-		"typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck",
-		"web": "pnpm --filter @supabase-evals/web dev",
-		"web:build": "pnpm --filter @supabase-evals/web build",
-		"demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp",
-		"demo:executor": "pnpm --filter @supabase-evals/framework demo:executor",
-		"format": "biome check --write . && pnpm --filter @supabase-evals/web format",
-		"format:check": "biome check . && pnpm --filter @supabase-evals/web format:check",
-		"local": "pnpm --filter @supabase-evals/framework local"
-	},
-	"dependencies": {
-		"@ai-sdk/anthropic": "catalog:",
-		"@ai-sdk/openai": "catalog:",
-		"common-tags": "^1.8.2"
-	},
-	"devDependencies": {
-		"@biomejs/biome": "1.9.4",
-		"@supabase-evals/core": "workspace:*",
-		"@supabase-evals/sandbox": "workspace:*",
-		"@types/common-tags": "^1.8.4",
-		"@types/node": "catalog:",
-		"typescript": "catalog:"
-	},
-	"engines": {
-		"node": ">=22",
-		"pnpm": "10.24"
-	},
-	"packageManager": "pnpm@10.24.0"
+  "name": "supabase-evals",
+  "private": true,
+  "type": "module",
+  "workspaces": ["apps/*", "packages/*"],
+  "scripts": {
+    "check": "pnpm --filter @supabase-evals/framework check && pnpm --filter @supabase-evals/core test && pnpm --filter @supabase-evals/sandbox test",
+    "eval": "pnpm --filter @supabase-evals/framework eval",
+    "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry",
+    "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke",
+    "eval:force": "pnpm --filter @supabase-evals/framework eval:force",
+    "test:framework": "pnpm --filter @supabase-evals/framework test:framework",
+    "export-results": "pnpm --filter @supabase-evals/framework export-results",
+    "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck",
+    "web": "pnpm --filter @supabase-evals/web dev",
+    "web:build": "pnpm --filter @supabase-evals/web build",
+    "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp",
+    "demo:executor": "pnpm --filter @supabase-evals/framework demo:executor",
+    "format": "biome check --write . && pnpm --filter @supabase-evals/web format",
+    "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check",
+    "local": "pnpm --filter @supabase-evals/framework local"
+  },
+  "dependencies": {
+    "@ai-sdk/anthropic": "catalog:",
+    "@ai-sdk/openai": "catalog:",
+    "common-tags": "^1.8.2"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "1.9.4",
+    "@supabase-evals/core": "workspace:*",
+    "@supabase-evals/sandbox": "workspace:*",
+    "@types/common-tags": "^1.8.4",
+    "@types/node": "catalog:",
+    "typescript": "catalog:"
+  },
+  "engines": {
+    "node": ">=22",
+    "pnpm": "10.24"
+  },
+  "packageManager": "pnpm@10.24.0"
 }

From f875dbd773757781e9f8db0f4cd9518054294812 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 11:23:04 +0200
Subject: [PATCH 07/24] refactor: fallow review fixes on the local runner

- Decompose the two functions over the cognitive-complexity threshold:
  cmdRunOrCompare 36 -> 7 (buildOverrideEnv + runTreatment + reportComparison)
  and resolveBaselines 28 -> under threshold (loadPublishedFile +
  scanFileRows + refuseMissingBaselines), with named PublishedFile type.
- Suppress the four runtime-loaded docs files (content-api-server,
  sentry stubs) as unused-file false positives with reasons: they are
  spawned/injected at runtime (tsx argv, NODE_OPTIONS --import,
  module.register), never statically imported; same for the loader-hook
  resolve export.

fallow audit vs origin/main after this: introduced dead-code 0,
introduced duplication 0, no introduced function above the cognitive
threshold. Remaining audit findings are inherited (main's unused deps,
run-eval complexity, pre-existing dup groups) or CRAP-from-zero-coverage
on script files, consistent with the repo baseline. Behavior verified:
typecheck clean, smoke-local 8/8, experiments/compare output unchanged.
---
 .../scripts/docs/content-api-server.ts        |   1 +
 .../scripts/docs/sentry-stub-loader.mjs       |   2 +
 .../scripts/docs/sentry-stub-register.mjs     |   1 +
 apps/framework/scripts/docs/sentry-stub.mjs   |   1 +
 apps/framework/scripts/local.ts               | 285 ++++++++++++------
 5 files changed, 192 insertions(+), 98 deletions(-)

diff --git a/apps/framework/scripts/docs/content-api-server.ts b/apps/framework/scripts/docs/content-api-server.ts
index f0adbada..9e2ece16 100644
--- a/apps/framework/scripts/docs/content-api-server.ts
+++ b/apps/framework/scripts/docs/content-api-server.ts
@@ -1,3 +1,4 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
 /**
  * Standalone docs content GraphQL API for `search_docs`.
  *
diff --git a/apps/framework/scripts/docs/sentry-stub-loader.mjs b/apps/framework/scripts/docs/sentry-stub-loader.mjs
index da2d7b19..154e301f 100644
--- a/apps/framework/scripts/docs/sentry-stub-loader.mjs
+++ b/apps/framework/scripts/docs/sentry-stub-loader.mjs
@@ -1,6 +1,8 @@
+// fallow-ignore-file unused-file -- registered at runtime by sentry-stub-register.mjs via module.register()
 // Loader-thread resolve hook: '@sentry/nextjs' -> the no-op stub.
 const stubUrl = new URL('./sentry-stub.mjs', import.meta.url).href;
 
+// fallow-ignore-next-line unused-export -- Node loader-hook contract: the module system calls `resolve`
 export async function resolve(specifier, context, next) {
   if (specifier === '@sentry/nextjs') {
     return { url: stubUrl, shortCircuit: true };
diff --git a/apps/framework/scripts/docs/sentry-stub-register.mjs b/apps/framework/scripts/docs/sentry-stub-register.mjs
index 9d4bc57c..0c425d1b 100644
--- a/apps/framework/scripts/docs/sentry-stub-register.mjs
+++ b/apps/framework/scripts/docs/sentry-stub-register.mjs
@@ -1,3 +1,4 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
 // Registers a resolve hook that short-circuits '@sentry/nextjs' to the local
 // no-op stub. Injected via NODE_OPTIONS from `pnpm local docs api`; chains with tsx's
 // own hooks (ours only intercepts the one specifier). Uses module.register()
diff --git a/apps/framework/scripts/docs/sentry-stub.mjs b/apps/framework/scripts/docs/sentry-stub.mjs
index 6eb3c60f..d42bb83f 100644
--- a/apps/framework/scripts/docs/sentry-stub.mjs
+++ b/apps/framework/scripts/docs/sentry-stub.mjs
@@ -1,3 +1,4 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
 // No-op @sentry/nextjs stand-in for the standalone docs content API.
 // The route handler calls Sentry.captureException/flush; under plain tsx
 // (outside Next's Sentry instrumentation) the real package's ESM build
diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts
index 3f7e704c..a3c8ceaa 100755
--- a/apps/framework/scripts/local.ts
+++ b/apps/framework/scripts/local.ts
@@ -155,6 +155,91 @@ type Baseline = {
   committedAt: string;
 };
 
+type PublishedFile = {
+  file: string;
+  rows: PublishedRow[];
+  commit: string;
+  parent: string;
+  committedAt: string;
+};
+
+/** Load one published export file from origin/main with its commit metadata. */
+function loadPublishedFile(file: string): PublishedFile | undefined {
+  let rows: PublishedRow[];
+  try {
+    rows = JSON.parse(git(['show', `origin/main:${file}`]));
+  } catch {
+    return undefined;
+  }
+  const [commit, parent, committedAt] = git([
+    'log',
+    'origin/main',
+    '-1',
+    '--format=%H %P %cI',
+    '--',
+    file,
+  ]).split(' ');
+  return { file, rows, commit, parent, committedAt };
+}
+
+/** Report evals with no published row for the experiment, then exit. */
+function refuseMissingBaselines(
+  missing: string[],
+  experiment: string,
+  seen: Map>
+): never {
+  for (const e of missing) {
+    const alts = [...(seen.get(e) ?? [])];
+    console.error(
+      alts.length
+        ? `no published ${experiment} result for ${e} on origin/main (published experiments: ${alts.join(', ')})`
+        : `no published result for ${e} on origin/main at all — use \`pnpm local run\` (no baseline needed)`
+    );
+  }
+  process.exit(1);
+}
+
+/** Fold one published file's rows into best/seen for the requested evals. */
+function scanFileRows(
+  loaded: PublishedFile,
+  evalIds: string[],
+  experiment: string,
+  best: Map,
+  seen: Map>
+) {
+  const { rows, commit, parent, committedAt } = loaded;
+  for (const row of rows) {
+    if (!evalIds.includes(row.eval)) continue;
+    const experiments = seen.get(row.eval) ?? new Set();
+    experiments.add(row.experiment);
+    seen.set(row.eval, experiments);
+    if (row.experiment !== experiment) continue;
+    const cur = best.get(row.eval);
+    if (!cur || new Date(committedAt) > new Date(cur.committedAt))
+      best.set(row.eval, {
+        row,
+        file: loaded.file,
+        commit,
+        parent,
+        committedAt,
+      });
+  }
+}
+
+/** Scan the published export files: freshest matching row per eval, plus every experiment seen per eval. */
+function scanPublishedRows(
+  evalIds: string[],
+  experiment: string
+): { best: Map; seen: Map> } {
+  const best = new Map();
+  const seen = new Map>();
+  for (const file of PUBLISHED_FILES) {
+    const loaded = loadPublishedFile(file);
+    if (loaded) scanFileRows(loaded, evalIds, experiment, best, seen);
+  }
+  return { best, seen };
+}
+
 function resolveBaselines(
   evalIds: string[],
   experiment: string,
@@ -169,45 +254,9 @@ function resolveBaselines(
       );
     }
   }
-  const best = new Map();
-  const seen = new Map>();
-  for (const file of PUBLISHED_FILES) {
-    let rows: PublishedRow[];
-    try {
-      rows = JSON.parse(git(['show', `origin/main:${file}`]));
-    } catch {
-      continue;
-    }
-    const [commit, parent, committedAt] = git([
-      'log',
-      'origin/main',
-      '-1',
-      '--format=%H %P %cI',
-      '--',
-      file,
-    ]).split(' ');
-    for (const row of rows) {
-      if (!evalIds.includes(row.eval)) continue;
-      if (!seen.has(row.eval)) seen.set(row.eval, new Set());
-      seen.get(row.eval)?.add(row.experiment);
-      if (row.experiment !== experiment) continue;
-      const cur = best.get(row.eval);
-      if (!cur || new Date(committedAt) > new Date(cur.committedAt))
-        best.set(row.eval, { row, file, commit, parent, committedAt });
-    }
-  }
+  const { best, seen } = scanPublishedRows(evalIds, experiment);
   const missing = evalIds.filter((e) => !best.has(e));
-  if (missing.length) {
-    for (const e of missing) {
-      const alts = [...(seen.get(e) ?? [])];
-      console.error(
-        alts.length
-          ? `no published ${experiment} result for ${e} on origin/main (published experiments: ${alts.join(', ')})`
-          : `no published result for ${e} on origin/main at all — use \`pnpm local run\` (no baseline needed)`
-      );
-    }
-    process.exit(1);
-  }
+  if (missing.length) refuseMissingBaselines(missing, experiment, seen);
   return best;
 }
 
@@ -323,6 +372,60 @@ async function cmdExperiments() {
   }
 }
 
+/** Resolve --mcp / --content-api into the child env, validating paths pre-spend. */
+function buildOverrideEnv(flags: Map): {
+  env: Record;
+  mcpPath?: string;
+  contentApi?: string;
+} {
+  const env: Record = {};
+  let mcpPath = flags.get('mcp');
+  if (mcpPath) {
+    mcpPath = isAbsolute(mcpPath) ? mcpPath : resolve(process.cwd(), mcpPath);
+    if (!existsSync(mcpPath)) fail(`--mcp path does not exist: ${mcpPath}`);
+    env.SUPABASE_MCP_SERVER_PATH = mcpPath;
+  }
+  const contentApi = flags.get('content-api');
+  if (contentApi) env.SUPABASE_CONTENT_API_URL = contentApi;
+  return { env, mcpPath, contentApi };
+}
+
+/** Print the published-vs-treatment delta; true when treatment regressed. */
+function reportComparison(
+  id: string,
+  b: Baseline,
+  result: PublishedRow
+): boolean {
+  writeFileSync(
+    join(OUT_DIR, `${id}.published.json`),
+    `${JSON.stringify({ ...b.row, publishedProvenance: { file: b.file, commit: b.commit, parent: b.parent, committedAt: b.committedAt } }, null, 1)}\n`
+  );
+  const ageDays = Math.round(
+    (Date.now() - new Date(b.committedAt).getTime()) / 86_400_000
+  );
+  console.log(
+    reportRow(
+      'published',
+      b.row,
+      `main@${b.commit.slice(0, 7)} ${b.committedAt.slice(0, 10)} (${ageDays}d old, attempts ${b.row.attempts})`
+    )
+  );
+  console.log(reportRow('treatment', result, 'your world'));
+  const d = (result.passed ? 1 : 0) - (b.row.passed ? 1 : 0);
+  console.log(
+    d > 0
+      ? '-> IMPROVED vs published (FAIL->PASS)'
+      : d < 0
+        ? '-> REGRESSED vs published (PASS->FAIL)'
+        : '-> no pass/fail change (compare checks / docs.calls)'
+  );
+  console.log(
+    'screen only: the published arm ran in the scheduled CI world — a flip is a signal, not causal proof'
+  );
+  console.log(`saved: results-local/${id}.{published,treatment}.json`);
+  return d < 0;
+}
+
 function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
   const { flags, positionals } = parseArgs(argv);
   const evalIds = positionals;
@@ -339,16 +442,7 @@ function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
       : new Map();
 
   validateEvals(evalIds);
-
-  const env: Record = {};
-  let mcpPath = flags.get('mcp');
-  if (mcpPath) {
-    mcpPath = isAbsolute(mcpPath) ? mcpPath : resolve(process.cwd(), mcpPath);
-    if (!existsSync(mcpPath)) fail(`--mcp path does not exist: ${mcpPath}`);
-    env.SUPABASE_MCP_SERVER_PATH = mcpPath;
-  }
-  const contentApi = flags.get('content-api');
-  if (contentApi) env.SUPABASE_CONTENT_API_URL = contentApi;
+  const { env, mcpPath, contentApi } = buildOverrideEnv(flags);
 
   mkdirSync(OUT_DIR, { recursive: true });
   let exitCode = 0;
@@ -356,60 +450,55 @@ function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
     const runs = Number(
       flags.get('runs') ?? baselines.get(id)?.row.attempts ?? 1
     );
-    console.log(
-      `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}${contentApi ? ', content-api override' : ''}) ==`
-    );
-    const resultPath = process.env.LOCAL_EVAL_CMD
-      ? fakeRun(id, experiment)
-      : runEval(id, experiment, runs, env);
-
-    const result = JSON.parse(readFileSync(resultPath, 'utf8')) as PublishedRow;
-    const receipt = {
-      ...result,
-      provenance: collectProvenance(mcpPath, contentApi),
-    };
-    const treatmentPath = join(OUT_DIR, `${id}.treatment.json`);
-    writeFileSync(treatmentPath, `${JSON.stringify(receipt, null, 1)}\n`);
-
-    console.log(`\n=== local ${mode}: ${id} (${experiment}) ===`);
-    const b = baselines.get(id);
-    if (b) {
-      writeFileSync(
-        join(OUT_DIR, `${id}.published.json`),
-        `${JSON.stringify({ ...b.row, publishedProvenance: { file: b.file, commit: b.commit, parent: b.parent, committedAt: b.committedAt } }, null, 1)}\n`
-      );
-      const ageDays = Math.round(
-        (Date.now() - new Date(b.committedAt).getTime()) / 86_400_000
-      );
-      console.log(
-        reportRow(
-          'published',
-          b.row,
-          `main@${b.commit.slice(0, 7)} ${b.committedAt.slice(0, 10)} (${ageDays}d old, attempts ${b.row.attempts})`
-        )
-      );
-      console.log(reportRow('treatment', result, 'your world'));
-      const d = (result.passed ? 1 : 0) - (b.row.passed ? 1 : 0);
-      console.log(
-        d > 0
-          ? '-> IMPROVED vs published (FAIL->PASS)'
-          : d < 0
-            ? '-> REGRESSED vs published (PASS->FAIL)'
-            : '-> no pass/fail change (compare checks / docs.calls)'
-      );
-      if (d < 0) exitCode = 1;
-      console.log(
-        'screen only: the published arm ran in the scheduled CI world — a flip is a signal, not causal proof'
-      );
-      console.log(`saved: results-local/${id}.{published,treatment}.json`);
-    } else {
-      console.log(reportRow('treatment', result, 'your world'));
-      console.log(`saved: results-local/${id}.treatment.json`);
-    }
+    const regressed = runTreatment(mode, id, experiment, runs, {
+      env,
+      mcpPath,
+      contentApi,
+      baseline: baselines.get(id),
+    });
+    if (regressed) exitCode = 1;
   }
   process.exit(exitCode);
 }
 
+/** Run one eval in the treatment world, write its receipt, report; true when it regressed vs published. */
+function runTreatment(
+  mode: 'run' | 'compare',
+  id: string,
+  experiment: string,
+  runs: number,
+  opts: {
+    env: Record;
+    mcpPath?: string;
+    contentApi?: string;
+    baseline?: Baseline;
+  }
+): boolean {
+  const { env, mcpPath, contentApi, baseline } = opts;
+  console.log(
+    `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}${contentApi ? ', content-api override' : ''}) ==`
+  );
+  const resultPath = process.env.LOCAL_EVAL_CMD
+    ? fakeRun(id, experiment)
+    : runEval(id, experiment, runs, env);
+
+  const result = JSON.parse(readFileSync(resultPath, 'utf8')) as PublishedRow;
+  const receipt = {
+    ...result,
+    provenance: collectProvenance(mcpPath, contentApi),
+  };
+  writeFileSync(
+    join(OUT_DIR, `${id}.treatment.json`),
+    `${JSON.stringify(receipt, null, 1)}\n`
+  );
+
+  console.log(`\n=== local ${mode}: ${id} (${experiment}) ===`);
+  if (baseline) return reportComparison(id, baseline, result);
+  console.log(reportRow('treatment', result, 'your world'));
+  console.log(`saved: results-local/${id}.treatment.json`);
+  return false;
+}
+
 // test hook: LOCAL_EVAL_CMD writes the result file itself (no model spend)
 function fakeRun(evalId: string, experiment: string): string {
   const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`);

From 1e52c098b19c2ca9bf690dd9d920f8e764fb6461 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 11:35:18 +0200
Subject: [PATCH 08/24] refactor: code-quality pass on the local runner
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Four structural fixes from a strict self-review:

- Delete both bespoke arg parsers (local.ts parseArgs, local-docs.ts flag
  loop) for node:util parseArgs — two fewer concepts, typed values without
  casts, and unknown flags now fail loudly instead of being silently
  swallowed.
- Delete the hand-rolled PublishedRow shape for the repo's own canonical
  contract: rawEvalResultSchema (+ new RawEvalResult type exported from
  eval-metadata, the module that owns the schema). Treatment results are
  now VALIDATED against the contract instead of blind-cast, so a malformed
  result file fails with a schema error rather than propagating undefined.
- Restructure resolveBaselines declaratively (per-eval candidates ->
  freshest match), deleting the scanFileRows accumulator threading and the
  seen-map bookkeeping.
- Drop the redundant mode param from runTreatment (baseline presence
  already encodes it).

Behavior identical: typecheck clean, core 57/57, smoke-local 8/8, error
paths verified verbatim. fallow: introduced dead-code 0, duplication 0,
nothing over complexity thresholds.
---
 apps/framework/scripts/local-docs.ts |  64 ++---
 apps/framework/scripts/local.ts      | 363 +++++++++++----------------
 packages/core/src/eval-metadata.ts   |   1 +
 3 files changed, 188 insertions(+), 240 deletions(-)

diff --git a/apps/framework/scripts/local-docs.ts b/apps/framework/scripts/local-docs.ts
index a52c3a0a..58b6f9ab 100644
--- a/apps/framework/scripts/local-docs.ts
+++ b/apps/framework/scripts/local-docs.ts
@@ -37,6 +37,7 @@ import {
 import { createInterface } from 'node:readline/promises';
 import { dirname, isAbsolute, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = resolve(__dirname, '..', '..', '..');
@@ -52,6 +53,8 @@ const STACK_EXCLUDES =
 
 const onWindows = process.platform === 'win32';
 
+type DocsOptions = { docs?: string; port?: string; yes?: boolean };
+
 function fail(msg: string): never {
   console.error(msg);
   process.exit(1);
@@ -82,10 +85,10 @@ function capture(cmd: string, args: string[]): string {
   return execFileSync(cmd, args, { cwd: ROOT, maxBuffer: 1 << 24 }).toString();
 }
 
-function docsPath(flags: Map): string {
+function docsPath(docs: string | undefined): string {
   const marker = join(OVERLAY, 'docs-path.txt');
   let p =
-    flags.get('docs') ??
+    docs ??
     (existsSync(marker) ? readFileSync(marker, 'utf8').trim() : undefined);
   if (!p)
     fail(
@@ -113,8 +116,8 @@ function stackEnv(): Record {
   return env;
 }
 
-function cmdUp(flags: Map) {
-  const docs = docsPath(flags);
+function cmdUp(opts: DocsOptions) {
+  const docs = docsPath(opts.docs);
   const src = join(docs, 'supabase');
   existsSync(join(src, 'config.toml')) ||
     fail(`no supabase/config.toml in the docs checkout: ${docs}`);
@@ -159,8 +162,8 @@ function cmdUp(flags: Map) {
   );
 }
 
-async function cmdSeed(flags: Map) {
-  const docs = docsPath(flags);
+async function cmdSeed(opts: DocsOptions) {
+  const docs = docsPath(opts.docs);
   if (!process.env.OPENAI_API_KEY)
     fail('OPENAI_API_KEY not set — add it to .env at the repo root');
   const docsApp = join(docs, 'apps', 'docs');
@@ -170,7 +173,7 @@ async function cmdSeed(flags: Map) {
     );
   }
   const env = stackEnv();
-  if (!flags.has('yes') && !process.env.LOCAL_DOCS_YES) {
+  if (!opts.yes && !process.env.LOCAL_DOCS_YES) {
     const rl = createInterface({
       input: process.stdin,
       output: process.stdout,
@@ -197,10 +200,10 @@ async function cmdSeed(flags: Map) {
   );
 }
 
-function cmdApi(flags: Map) {
-  const docs = docsPath(flags);
+function cmdApi(opts: DocsOptions) {
+  const docs = docsPath(opts.docs);
   const docsApp = join(docs, 'apps', 'docs');
-  const port = flags.get('port') ?? '3001';
+  const port = opts.port ?? '3001';
   const env = stackEnv();
   const tsx = join(
     docsApp,
@@ -244,35 +247,38 @@ function cmdApi(flags: Map) {
 }
 
 export async function main(argv: string[]) {
-  const [sub, ...rest] = argv;
-  const flags = new Map();
-  const boolFlags = new Set(['yes']);
-  for (let i = 0; i < rest.length; i++) {
-    const a = rest[i];
-    if (!a.startsWith('--')) continue;
-    const name = a.slice(2);
-    if (boolFlags.has(name)) flags.set(name, '1');
-    else {
-      flags.set(name, rest[i + 1] ?? '');
-      i++;
+  const usage =
+    'usage: pnpm local docs  [--docs ] [--port N] [--yes]';
+  const parsed = (() => {
+    try {
+      return parseArgs({
+        args: argv,
+        options: {
+          docs: { type: 'string' },
+          port: { type: 'string' },
+          yes: { type: 'boolean' },
+        },
+        allowPositionals: true,
+      });
+    } catch (err) {
+      fail(`${err instanceof Error ? err.message : String(err)}\n${usage}`);
     }
-  }
-  switch (sub) {
+  })();
+  const { values } = parsed;
+  switch (parsed.positionals[0]) {
     case 'up':
-      cmdUp(flags);
+      cmdUp(values);
       break;
     case 'seed':
-      await cmdSeed(flags);
+      await cmdSeed(values);
       break;
     case 'api':
-      cmdApi(flags);
+      cmdApi(values);
       break;
     case 'down':
       run('supabase', ['stop', '--workdir', OVERLAY]);
       break;
     default:
-      fail(
-        'usage: pnpm local docs  [--docs ] [--port N] [--yes]'
-      );
+      fail(usage);
   }
 }
diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts
index a3c8ceaa..0cee5b97 100755
--- a/apps/framework/scripts/local.ts
+++ b/apps/framework/scripts/local.ts
@@ -31,11 +31,16 @@ import {
 } from 'node:fs';
 import { dirname, isAbsolute, join, resolve } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
-import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown';
+import { parseArgs, type ParseArgsConfig } from 'node:util';
 import {
   getExperimentDisplayMetadata,
   type ExperimentConfig,
 } from '@supabase-evals/core';
+import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown';
+import {
+  rawEvalResultSchema,
+  type RawEvalResult,
+} from '@supabase-evals/core/eval-metadata';
 import { main as docsMain } from './local-docs.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -47,34 +52,6 @@ const PUBLISHED_FILES = [
 ];
 const DEFAULT_EXPERIMENT = 'claude-code-sonnet-5';
 
-// ---------- tiny arg helpers (single-value flags; positionals collected) ----------
-
-function parseArgs(argv: string[]) {
-  const flags = new Map();
-  const bools = new Set();
-  const positionals: string[] = [];
-  for (let i = 0; i < argv.length; i++) {
-    const a = argv[i];
-    if (!a.startsWith('--')) {
-      positionals.push(a);
-      continue;
-    }
-    const eq = a.indexOf('=');
-    if (eq !== -1) {
-      flags.set(a.slice(2, eq), a.slice(eq + 1));
-      continue;
-    }
-    const next = argv[i + 1];
-    if (next !== undefined && !next.startsWith('--')) {
-      flags.set(a.slice(2), next);
-      i++;
-    } else {
-      bools.add(a.slice(2));
-    }
-  }
-  return { flags, bools, positionals };
-}
-
 function fail(msg: string): never {
   console.error(msg);
   process.exit(1);
@@ -124,11 +101,7 @@ function collectProvenance(mcpPath?: string, contentApi?: string): Provenance {
     p.mcpOverride = {
       path: mcpPath,
       sha: inRepo ? tryGit(['rev-parse', 'HEAD'], mcpPath) : undefined,
-      dirtyFiles: inRepo
-        ? (tryGit(['status', '--porcelain'], inRepo) ?? '')
-            .split('\n')
-            .filter(Boolean).length
-        : undefined,
+      dirtyFiles: inRepo ? dirty(inRepo) : undefined,
     };
   }
   if (contentApi) p.contentApiUrl = contentApi;
@@ -137,27 +110,17 @@ function collectProvenance(mcpPath?: string, contentApi?: string): Provenance {
 
 // ---------- published baselines (compare mode) ----------
 
-type PublishedRow = {
-  experiment: string;
-  eval: string;
-  passed?: boolean;
-  attempts?: number;
-  checks?: Array<{ name: string; passed: boolean }>;
-  docs?: { calls?: unknown[] };
-  [k: string]: unknown;
-};
-
-type Baseline = {
-  row: PublishedRow;
+type PublishedFile = {
   file: string;
+  rows: RawEvalResult[];
   commit: string;
   parent: string;
   committedAt: string;
 };
 
-type PublishedFile = {
+type Baseline = {
+  row: RawEvalResult;
   file: string;
-  rows: PublishedRow[];
   commit: string;
   parent: string;
   committedAt: string;
@@ -165,7 +128,7 @@ type PublishedFile = {
 
 /** Load one published export file from origin/main with its commit metadata. */
 function loadPublishedFile(file: string): PublishedFile | undefined {
-  let rows: PublishedRow[];
+  let rows: RawEvalResult[];
   try {
     rows = JSON.parse(git(['show', `origin/main:${file}`]));
   } catch {
@@ -182,64 +145,11 @@ function loadPublishedFile(file: string): PublishedFile | undefined {
   return { file, rows, commit, parent, committedAt };
 }
 
-/** Report evals with no published row for the experiment, then exit. */
-function refuseMissingBaselines(
-  missing: string[],
-  experiment: string,
-  seen: Map>
-): never {
-  for (const e of missing) {
-    const alts = [...(seen.get(e) ?? [])];
-    console.error(
-      alts.length
-        ? `no published ${experiment} result for ${e} on origin/main (published experiments: ${alts.join(', ')})`
-        : `no published result for ${e} on origin/main at all — use \`pnpm local run\` (no baseline needed)`
-    );
-  }
-  process.exit(1);
-}
-
-/** Fold one published file's rows into best/seen for the requested evals. */
-function scanFileRows(
-  loaded: PublishedFile,
-  evalIds: string[],
-  experiment: string,
-  best: Map,
-  seen: Map>
-) {
-  const { rows, commit, parent, committedAt } = loaded;
-  for (const row of rows) {
-    if (!evalIds.includes(row.eval)) continue;
-    const experiments = seen.get(row.eval) ?? new Set();
-    experiments.add(row.experiment);
-    seen.set(row.eval, experiments);
-    if (row.experiment !== experiment) continue;
-    const cur = best.get(row.eval);
-    if (!cur || new Date(committedAt) > new Date(cur.committedAt))
-      best.set(row.eval, {
-        row,
-        file: loaded.file,
-        commit,
-        parent,
-        committedAt,
-      });
-  }
-}
-
-/** Scan the published export files: freshest matching row per eval, plus every experiment seen per eval. */
-function scanPublishedRows(
-  evalIds: string[],
-  experiment: string
-): { best: Map; seen: Map> } {
-  const best = new Map();
-  const seen = new Map>();
-  for (const file of PUBLISHED_FILES) {
-    const loaded = loadPublishedFile(file);
-    if (loaded) scanFileRows(loaded, evalIds, experiment, best, seen);
-  }
-  return { best, seen };
-}
-
+/**
+ * Freshest published row per requested eval for the experiment. Refuses
+ * (pre-spend) when any requested eval has no published row, listing the
+ * experiments that ARE published for it.
+ */
 function resolveBaselines(
   evalIds: string[],
   experiment: string,
@@ -254,9 +164,34 @@ function resolveBaselines(
       );
     }
   }
-  const { best, seen } = scanPublishedRows(evalIds, experiment);
-  const missing = evalIds.filter((e) => !best.has(e));
-  if (missing.length) refuseMissingBaselines(missing, experiment, seen);
+  const files = PUBLISHED_FILES.flatMap((f) => loadPublishedFile(f) ?? []);
+  const best = new Map();
+  const failures: string[] = [];
+  for (const id of evalIds) {
+    const candidates: Baseline[] = files.flatMap(
+      ({ file, rows, commit, parent, committedAt }) =>
+        rows
+          .filter((row) => row.eval === id)
+          .map((row) => ({ row, file, commit, parent, committedAt }))
+    );
+    const match = candidates
+      .filter((c) => c.row.experiment === experiment)
+      .sort((a, b) => Date.parse(b.committedAt) - Date.parse(a.committedAt))[0];
+    if (match) {
+      best.set(id, match);
+      continue;
+    }
+    const alts = [...new Set(candidates.map((c) => c.row.experiment))];
+    failures.push(
+      alts.length
+        ? `no published ${experiment} result for ${id} on origin/main (published experiments: ${alts.join(', ')})`
+        : `no published result for ${id} on origin/main at all — use \`pnpm local run\` (no baseline needed)`
+    );
+  }
+  if (failures.length) {
+    for (const msg of failures) console.error(msg);
+    process.exit(1);
+  }
   return best;
 }
 
@@ -327,11 +262,24 @@ function runEval(
   return resultPath;
 }
 
+// test hook: LOCAL_EVAL_CMD writes the result file itself (no model spend)
+function fakeRun(evalId: string, experiment: string): string {
+  const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`);
+  mkdirSync(dirname(resultPath), { recursive: true });
+  const res = spawnSync(process.env.LOCAL_EVAL_CMD as string, {
+    shell: true,
+    stdio: 'inherit',
+    env: { ...process.env, RES: resultPath, EVAL: evalId },
+  });
+  if (res.status !== 0) fail(`LOCAL_EVAL_CMD failed for ${evalId}`);
+  return resultPath;
+}
+
 // ---------- reporting ----------
 
 function reportRow(
   label: string,
-  r: PublishedRow | undefined,
+  r: RawEvalResult | undefined,
   extra: string
 ): string {
   const checks = r?.checks ?? [];
@@ -340,61 +288,11 @@ function reportRow(
   return `${label.padEnd(10)} passed=${String(r?.passed).padEnd(5)} checks=${checksSummary.padEnd(6)} docs.calls=${String(docsCalls).padEnd(3)} ${extra}`;
 }
 
-// ---------- subcommands ----------
-
-async function cmdExperiments() {
-  const published = new Set();
-  for (const file of PUBLISHED_FILES) {
-    try {
-      for (const row of JSON.parse(
-        git(['show', `origin/main:${file}`])
-      ) as PublishedRow[])
-        published.add(row.experiment);
-    } catch {
-      /* offline or file missing: published column degrades to '-' */
-    }
-  }
-  console.log(
-    `${'EXPERIMENT'.padEnd(36)} ${'AGENT'.padEnd(12)} ${'MODEL'.padEnd(22)} ${'EFFORT'.padEnd(8)} PUBLISHED`
-  );
-  for (const f of readdirSync(join(ROOT, 'experiments'))
-    .filter((f) => f.endsWith('.ts'))
-    .sort()) {
-    const name = f.replace(/\.ts$/, '');
-    // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments)
-    const mod = await import(pathToFileURL(join(ROOT, 'experiments', f)).href);
-    const display = getExperimentDisplayMetadata(
-      mod.default as ExperimentConfig
-    );
-    console.log(
-      `${name.padEnd(36)} ${(display.agent ?? '?').padEnd(12)} ${(display.modelId ?? '?').padEnd(22)} ${(display.reasoningEffort ?? '-').padEnd(8)} ${published.has(name) ? 'yes (compare)' : '-'}`
-    );
-  }
-}
-
-/** Resolve --mcp / --content-api into the child env, validating paths pre-spend. */
-function buildOverrideEnv(flags: Map): {
-  env: Record;
-  mcpPath?: string;
-  contentApi?: string;
-} {
-  const env: Record = {};
-  let mcpPath = flags.get('mcp');
-  if (mcpPath) {
-    mcpPath = isAbsolute(mcpPath) ? mcpPath : resolve(process.cwd(), mcpPath);
-    if (!existsSync(mcpPath)) fail(`--mcp path does not exist: ${mcpPath}`);
-    env.SUPABASE_MCP_SERVER_PATH = mcpPath;
-  }
-  const contentApi = flags.get('content-api');
-  if (contentApi) env.SUPABASE_CONTENT_API_URL = contentApi;
-  return { env, mcpPath, contentApi };
-}
-
 /** Print the published-vs-treatment delta; true when treatment regressed. */
 function reportComparison(
   id: string,
   b: Baseline,
-  result: PublishedRow
+  result: RawEvalResult
 ): boolean {
   writeFileSync(
     join(OUT_DIR, `${id}.published.json`),
@@ -426,44 +324,8 @@ function reportComparison(
   return d < 0;
 }
 
-function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
-  const { flags, positionals } = parseArgs(argv);
-  const evalIds = positionals;
-  if (!evalIds.length)
-    fail(
-      `usage: pnpm local ${mode}  [...] [--experiment ] [--runs N] [--mcp ] [--content-api ]`
-    );
-  const experiment = flags.get('experiment') ?? DEFAULT_EXPERIMENT;
-  validateExperiment(experiment);
-
-  const baselines =
-    mode === 'compare'
-      ? resolveBaselines(evalIds, experiment, !process.env.LOCAL_NO_FETCH)
-      : new Map();
-
-  validateEvals(evalIds);
-  const { env, mcpPath, contentApi } = buildOverrideEnv(flags);
-
-  mkdirSync(OUT_DIR, { recursive: true });
-  let exitCode = 0;
-  for (const id of evalIds) {
-    const runs = Number(
-      flags.get('runs') ?? baselines.get(id)?.row.attempts ?? 1
-    );
-    const regressed = runTreatment(mode, id, experiment, runs, {
-      env,
-      mcpPath,
-      contentApi,
-      baseline: baselines.get(id),
-    });
-    if (regressed) exitCode = 1;
-  }
-  process.exit(exitCode);
-}
-
 /** Run one eval in the treatment world, write its receipt, report; true when it regressed vs published. */
 function runTreatment(
-  mode: 'run' | 'compare',
   id: string,
   experiment: string,
   runs: number,
@@ -482,7 +344,14 @@ function runTreatment(
     ? fakeRun(id, experiment)
     : runEval(id, experiment, runs, env);
 
-  const result = JSON.parse(readFileSync(resultPath, 'utf8')) as PublishedRow;
+  const parsed = rawEvalResultSchema.safeParse(
+    JSON.parse(readFileSync(resultPath, 'utf8'))
+  );
+  if (!parsed.success)
+    fail(
+      `result at ${resultPath} does not match the eval result contract:\n${parsed.error.message}`
+    );
+  const result = parsed.data;
   const receipt = {
     ...result,
     provenance: collectProvenance(mcpPath, contentApi),
@@ -492,24 +361,96 @@ function runTreatment(
     `${JSON.stringify(receipt, null, 1)}\n`
   );
 
-  console.log(`\n=== local ${mode}: ${id} (${experiment}) ===`);
+  console.log(
+    `\n=== local ${baseline ? 'compare' : 'run'}: ${id} (${experiment}) ===`
+  );
   if (baseline) return reportComparison(id, baseline, result);
   console.log(reportRow('treatment', result, 'your world'));
   console.log(`saved: results-local/${id}.treatment.json`);
   return false;
 }
 
-// test hook: LOCAL_EVAL_CMD writes the result file itself (no model spend)
-function fakeRun(evalId: string, experiment: string): string {
-  const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`);
-  mkdirSync(dirname(resultPath), { recursive: true });
-  const res = spawnSync(process.env.LOCAL_EVAL_CMD as string, {
-    shell: true,
-    stdio: 'inherit',
-    env: { ...process.env, RES: resultPath, EVAL: evalId },
-  });
-  if (res.status !== 0) fail(`LOCAL_EVAL_CMD failed for ${evalId}`);
-  return resultPath;
+// ---------- subcommands ----------
+
+async function cmdExperiments() {
+  const published = new Set(
+    PUBLISHED_FILES.flatMap(
+      (f) => loadPublishedFile(f)?.rows.map((r) => r.experiment) ?? []
+    )
+  );
+  console.log(
+    `${'EXPERIMENT'.padEnd(36)} ${'AGENT'.padEnd(12)} ${'MODEL'.padEnd(22)} ${'EFFORT'.padEnd(8)} PUBLISHED`
+  );
+  for (const f of readdirSync(join(ROOT, 'experiments'))
+    .filter((f) => f.endsWith('.ts'))
+    .sort()) {
+    const name = f.replace(/\.ts$/, '');
+    // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments)
+    const mod = await import(pathToFileURL(join(ROOT, 'experiments', f)).href);
+    const display = getExperimentDisplayMetadata(
+      mod.default as ExperimentConfig
+    );
+    console.log(
+      `${name.padEnd(36)} ${(display.agent ?? '?').padEnd(12)} ${(display.modelId ?? '?').padEnd(22)} ${(display.reasoningEffort ?? '-').padEnd(8)} ${published.has(name) ? 'yes (compare)' : '-'}`
+    );
+  }
+}
+
+const RUN_USAGE =
+  'usage: pnpm local   [...] [--experiment ] [--runs N] [--mcp ] [--content-api ]';
+
+function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
+  const parsed = (() => {
+    try {
+      return parseArgs({
+        args: argv,
+        options: {
+          experiment: { type: 'string' },
+          runs: { type: 'string' },
+          mcp: { type: 'string' },
+          'content-api': { type: 'string' },
+        },
+        allowPositionals: true,
+      });
+    } catch (err) {
+      fail(`${err instanceof Error ? err.message : String(err)}\n${RUN_USAGE}`);
+    }
+  })();
+  const { values, positionals: evalIds } = parsed;
+  if (!evalIds.length) fail(RUN_USAGE);
+  const experiment = values.experiment ?? DEFAULT_EXPERIMENT;
+  validateExperiment(experiment);
+
+  const baselines =
+    mode === 'compare'
+      ? resolveBaselines(evalIds, experiment, !process.env.LOCAL_NO_FETCH)
+      : new Map();
+
+  validateEvals(evalIds);
+
+  const env: Record = {};
+  let mcpPath = values.mcp;
+  if (mcpPath) {
+    mcpPath = isAbsolute(mcpPath) ? mcpPath : resolve(process.cwd(), mcpPath);
+    if (!existsSync(mcpPath)) fail(`--mcp path does not exist: ${mcpPath}`);
+    env.SUPABASE_MCP_SERVER_PATH = mcpPath;
+  }
+  const contentApi = values['content-api'];
+  if (contentApi) env.SUPABASE_CONTENT_API_URL = contentApi;
+
+  mkdirSync(OUT_DIR, { recursive: true });
+  let exitCode = 0;
+  for (const id of evalIds) {
+    const runs = Number(values.runs ?? baselines.get(id)?.row.attempts ?? 1);
+    const regressed = runTreatment(id, experiment, runs, {
+      env,
+      mcpPath,
+      contentApi,
+      baseline: baselines.get(id),
+    });
+    if (regressed) exitCode = 1;
+  }
+  process.exit(exitCode);
 }
 
 // ---------- entry ----------
diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts
index 1a8d541c..323cdf5f 100644
--- a/packages/core/src/eval-metadata.ts
+++ b/packages/core/src/eval-metadata.ts
@@ -306,6 +306,7 @@ const evalResultShape = {
 
 // Raw result files may carry extra fields we don't model; tolerate them.
 export const rawEvalResultSchema = z.looseObject(evalResultShape);
+export type RawEvalResult = z.infer;
 
 // Web-facing result; a clean strict object so its inferred type stays usable.
 export const evalResultSchema = z.object({

From 29dc477a4686b8a736986e86106804a8362e3eb8 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 11:48:58 +0200
Subject: [PATCH 09/24] fix: --mcp accepts the checkout root; pre-spend gates
 for unbuilt server and missing skills
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two failures from the first manual validation run:

- The core resolver wants the server package dir, but every natural
  invocation passes the checkout root. resolveMcpServerPath now accepts
  either (root resolves to packages/mcp-server-supabase) and refuses
  pre-spend when dist/transports/stdio.js is missing, with the build
  command — previously the harness discovered that only inside eval setup.
- An uninitialised skills submodule made the harness warn-and-skip the
  experiment's declared skills: a treatment silently running skill-less
  against a skills-enabled published baseline is a world mismatch, not a
  comparison. validateSkills refuses pre-spend with the submodule command.

smoke-local: 10 checks (new: unbuilt-checkout refusal with hint, root ->
package-dir resolution asserted via the receipt's mcpOverride.path).
Proven against a real mcp checkout: root path resolves, receipt records
the canary commit sha.
---
 apps/framework/scripts/local.ts       | 49 ++++++++++++++++++++++-----
 apps/framework/scripts/smoke-local.ts | 42 ++++++++++++++++++++++-
 2 files changed, 82 insertions(+), 9 deletions(-)

diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts
index 0cee5b97..74b81b77 100755
--- a/apps/framework/scripts/local.ts
+++ b/apps/framework/scripts/local.ts
@@ -226,6 +226,41 @@ function validateExperiment(experiment: string) {
   }
 }
 
+/**
+ * Accept either the mcp monorepo root or the server package dir for --mcp,
+ * and refuse pre-spend when the server isn't built (the harness would only
+ * discover that after eval setup).
+ */
+function resolveMcpServerPath(raw: string): string {
+  let p = isAbsolute(raw) ? raw : resolve(process.cwd(), raw);
+  if (!existsSync(p)) fail(`--mcp path does not exist: ${p}`);
+  const packageDir = join(p, 'packages', 'mcp-server-supabase');
+  if (existsSync(packageDir)) p = packageDir;
+  if (!existsSync(join(p, 'dist', 'transports', 'stdio.js')))
+    fail(
+      `no built server at ${p} (dist/transports/stdio.js missing) — build it first:\n  pnpm install && pnpm build   # in the mcp checkout (use \`mise exec --\` if corepack's pnpm mismatches)`
+    );
+  return p;
+}
+
+/**
+ * The experiment's declared skills must exist in this checkout, or the
+ * treatment silently runs skill-less against a skills-enabled published
+ * baseline — a world mismatch, not a comparison.
+ */
+async function validateSkills(experiment: string) {
+  // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments)
+  const mod = await import(
+    pathToFileURL(join(ROOT, 'experiments', `${experiment}.ts`)).href
+  );
+  const skills: string[] = (mod.default as ExperimentConfig).skills ?? [];
+  const missing = skills.filter((s) => !existsSync(join(ROOT, 'skills', s)));
+  if (missing.length)
+    fail(
+      `experiment ${experiment} declares skills this checkout is missing: ${missing.join(', ')}\ninitialise the skills submodule first: git submodule update --init`
+    );
+}
+
 // ---------- treatment run ----------
 
 function runEval(
@@ -399,7 +434,7 @@ async function cmdExperiments() {
 const RUN_USAGE =
   'usage: pnpm local   [...] [--experiment ] [--runs N] [--mcp ] [--content-api ]';
 
-function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
+async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
   const parsed = (() => {
     try {
       return parseArgs({
@@ -427,14 +462,12 @@ function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
       : new Map();
 
   validateEvals(evalIds);
+  // skills gate is spend-relevant only for real runs; the test hook fakes them
+  if (!process.env.LOCAL_EVAL_CMD) await validateSkills(experiment);
 
   const env: Record = {};
-  let mcpPath = values.mcp;
-  if (mcpPath) {
-    mcpPath = isAbsolute(mcpPath) ? mcpPath : resolve(process.cwd(), mcpPath);
-    if (!existsSync(mcpPath)) fail(`--mcp path does not exist: ${mcpPath}`);
-    env.SUPABASE_MCP_SERVER_PATH = mcpPath;
-  }
+  const mcpPath = values.mcp ? resolveMcpServerPath(values.mcp) : undefined;
+  if (mcpPath) env.SUPABASE_MCP_SERVER_PATH = mcpPath;
   const contentApi = values['content-api'];
   if (contentApi) env.SUPABASE_CONTENT_API_URL = contentApi;
 
@@ -459,7 +492,7 @@ const [command, ...rest] = process.argv.slice(2);
 switch (command) {
   case 'run':
   case 'compare':
-    cmdRunOrCompare(command, rest);
+    await cmdRunOrCompare(command, rest);
     break;
   case 'experiments':
     await cmdExperiments();
diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts
index 4dc549a4..8e4bbc5b 100644
--- a/apps/framework/scripts/smoke-local.ts
+++ b/apps/framework/scripts/smoke-local.ts
@@ -8,7 +8,13 @@
  */
 import assert from 'node:assert/strict';
 import { execFileSync, spawnSync } from 'node:child_process';
-import { existsSync, readFileSync, rmSync } from 'node:fs';
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
@@ -142,6 +148,40 @@ function ck(name: string, fn: () => void) {
   });
 }
 
+// --- mcp override: monorepo root resolves to the server package; unbuilt refused ---
+{
+  const fake = join(ROOT, 'results-local', '.smoke-mcp-checkout');
+  const pkg = join(fake, 'packages', 'mcp-server-supabase');
+  mkdirSync(join(pkg, 'dist', 'transports'), { recursive: true });
+
+  const unbuilt = local(['run', EVAL, '--mcp', fake]);
+  ck('unbuilt mcp checkout refused pre-spend with build hint', () => {
+    assert.equal(unbuilt.status, 1);
+    assert.match(unbuilt.out, /no built server at .*mcp-server-supabase/);
+    assert.match(unbuilt.out, /pnpm install && pnpm build/);
+  });
+
+  writeFileSync(
+    join(pkg, 'dist', 'transports', 'stdio.js'),
+    '// smoke fixture\n'
+  );
+  const built = local(['run', EVAL, '--mcp', fake]);
+  ck('monorepo root resolves to the server package dir', () => {
+    assert.equal(built.status, 0);
+    const receipt = JSON.parse(
+      readFileSync(
+        join(ROOT, 'results-local', `${EVAL}.treatment.json`),
+        'utf8'
+      )
+    );
+    assert.match(
+      receipt.provenance.mcpOverride.path,
+      /packages[/\\]mcp-server-supabase$/
+    );
+  });
+  rmSync(fake, { recursive: true, force: true });
+}
+
 // cleanup
 rmSync(join(ROOT, 'results-local', `${EVAL}.published.json`), { force: true });
 rmSync(join(ROOT, 'results-local', `${EVAL}.treatment.json`), { force: true });

From 7668ef5d2e99ac0be2afd761103960026cc68d1e Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 11:57:40 +0200
Subject: [PATCH 10/24] fix: pre-spend gate for judge-scored evals needing
 OPENAI_API_KEY
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Observed on a manual run: the agent (claude-code) completed its full paid
run, then scoring failed at assertProviderReady — the eval's scorer grades
with the LLM judge, which is an OpenAI model regardless of the agent under
test. 16 evals use the judge today. validateJudgeKeys now scans the
selected evals' scorers and refuses BEFORE any agent spawn when
OPENAI_API_KEY is missing, naming the affected evals.

smoke-local: 11 checks; the new one selects a known-judged eval from the
published set (the generic pick may be unjudged, which would reach a real
agent spawn — that was a test-design bug caught while writing it).
README/AGENTS key docs updated.
---
 AGENTS.md                             |  6 ++++--
 README.md                             |  5 +++--
 apps/framework/scripts/local.ts       | 25 +++++++++++++++++++++++--
 apps/framework/scripts/smoke-local.ts | 24 ++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 15ba1cb5..b6fbe330 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -65,8 +65,10 @@ arm's result commit + parent + age.
 
 ## Conventions
 
-- Keys live in `.env` at the repo root (`ANTHROPIC_API_KEY`; plus
-  `OPENAI_API_KEY` for the docs loop). Never hardcode or echo key values.
+- Keys live in `.env` at the repo root: `ANTHROPIC_API_KEY`, plus
+  `OPENAI_API_KEY` for the docs loop AND for any eval whose scorer uses the
+  LLM judge (an OpenAI grader model runs even when the agent under test is
+  Claude). Never hardcode or echo key values.
 - Model/agent selection = experiment id. To test an unlisted model, add a
   small `experiments/.ts` (copy an existing file's shape) rather than
   editing a published experiment in place.
diff --git a/README.md b/README.md
index ff190ab0..cd0c0632 100644
--- a/README.md
+++ b/README.md
@@ -110,8 +110,9 @@ arm's result commit, parent, and age — and a pass/fail flip against published
 is a **screen**, not causal proof: the published run happened in the scheduled
 CI world (published mcp package, prod docs index, model state at refresh time).
 
-Keys go in `.env` at the repo root (`ANTHROPIC_API_KEY`, and `OPENAI_API_KEY`
-for the docs loop). Zero-cost self-test: `pnpm --filter
+Keys go in `.env` at the repo root: `ANTHROPIC_API_KEY`, and `OPENAI_API_KEY`
+for the docs loop and for judge-scored evals (the LLM judge is an OpenAI
+grader model, regardless of the agent under test). Zero-cost self-test: `pnpm --filter
 @supabase-evals/framework test:local`.
 
 ## Eval Shape
diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts
index 74b81b77..bb5e96d7 100755
--- a/apps/framework/scripts/local.ts
+++ b/apps/framework/scripts/local.ts
@@ -226,6 +226,24 @@ function validateExperiment(experiment: string) {
   }
 }
 
+/**
+ * Evals whose scorer uses the LLM judge grade the agent's output with an
+ * OpenAI model — even when the agent under test is Claude. A missing grader
+ * key otherwise surfaces only AFTER the (paid) agent run, wasting it.
+ * Textual scan of EVAL.ts; a false positive just asks for a key early.
+ */
+function validateJudgeKeys(evalIds: string[]) {
+  if (process.env.OPENAI_API_KEY) return;
+  const judged = evalIds.filter((id) => {
+    const scorer = join(ROOT, 'evals', id, 'EVAL.ts');
+    return existsSync(scorer) && /\bjudge\b/.test(readFileSync(scorer, 'utf8'));
+  });
+  if (judged.length)
+    fail(
+      `these evals score with the LLM judge (OpenAI-backed, regardless of the agent under test): ${judged.join(', ')}\nadd OPENAI_API_KEY to .env at the repo root before running them`
+    );
+}
+
 /**
  * Accept either the mcp monorepo root or the server package dir for --mcp,
  * and refuse pre-spend when the server isn't built (the harness would only
@@ -462,8 +480,11 @@ async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
       : new Map();
 
   validateEvals(evalIds);
-  // skills gate is spend-relevant only for real runs; the test hook fakes them
-  if (!process.env.LOCAL_EVAL_CMD) await validateSkills(experiment);
+  // these gates are spend-relevant only for real runs; the test hook fakes them
+  if (!process.env.LOCAL_EVAL_CMD) {
+    await validateSkills(experiment);
+    validateJudgeKeys(evalIds);
+  }
 
   const env: Record = {};
   const mcpPath = values.mcp ? resolveMcpServerPath(values.mcp) : undefined;
diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts
index 8e4bbc5b..63685c01 100644
--- a/apps/framework/scripts/smoke-local.ts
+++ b/apps/framework/scripts/smoke-local.ts
@@ -49,6 +49,7 @@ function local(args: string[], env: Record = {}) {
     {
       cwd: join(__dirname, '..'),
       encoding: 'utf8',
+      timeout: 60_000, // a regressed pre-spend gate must never reach a real agent run
       env: {
         ...process.env,
         LOCAL_NO_FETCH: '1',
@@ -182,6 +183,29 @@ function ck(name: string, fn: () => void) {
   rmSync(fake, { recursive: true, force: true });
 }
 
+// --- judge-key gate: refused pre-spend, before any agent spawn ---
+{
+  // needs an eval whose scorer really uses the judge; EVAL may not
+  const judgedEval = published.find(
+    (row) =>
+      row.experiment === EXPERIMENT &&
+      existsSync(join(ROOT, 'evals', row.eval, 'EVAL.ts')) &&
+      /\bjudge\b/.test(
+        readFileSync(join(ROOT, 'evals', row.eval, 'EVAL.ts'), 'utf8')
+      )
+  )?.eval;
+  assert.ok(judgedEval, 'no judged eval found in the published set');
+  const r = local(['run', judgedEval], {
+    LOCAL_EVAL_CMD: '',
+    OPENAI_API_KEY: '',
+  });
+  ck('judged eval without OPENAI_API_KEY refused pre-spend', () => {
+    assert.equal(r.status, 1);
+    assert.match(r.out, /score with the LLM judge/);
+    assert.match(r.out, /add OPENAI_API_KEY/);
+  });
+}
+
 // cleanup
 rmSync(join(ROOT, 'results-local', `${EVAL}.published.json`), { force: true });
 rmSync(join(ROOT, 'results-local', `${EVAL}.treatment.json`), { force: true });

From c2dbc245a21675bd985ad08f63ca5b932c045a09 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 12:05:36 +0200
Subject: [PATCH 11/24] feat: --suite compare (published-set expansion);
 sandbox the smoke suite
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- pnpm local compare --suite  expands to every eval
  the published export carries for the experiment (printed with count
  before running: one model run each). Refused in run mode, with explicit
  ids, and for unknown suites.
- The smoke suite now redirects every output into a mkdtemp sandbox
  (LOCAL_RESULTS_ROOT seam). Previously it faked results and cleaned up
  under the checkout's real results/ + results-local/ paths — which can
  clobber an in-flight manual run's receipts for the same eval id.
  Observed live. smoke-local: 15 checks.
---
 apps/framework/scripts/local.ts       | 57 +++++++++++++++++++++++---
 apps/framework/scripts/smoke-local.ts | 58 +++++++++++++++++++--------
 2 files changed, 93 insertions(+), 22 deletions(-)

diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts
index bb5e96d7..e8f0c910 100755
--- a/apps/framework/scripts/local.ts
+++ b/apps/framework/scripts/local.ts
@@ -45,7 +45,10 @@ import { main as docsMain } from './local-docs.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = resolve(__dirname, '..', '..', '..');
-const OUT_DIR = join(ROOT, 'results-local');
+// test seam: the smoke suite redirects ALL outputs into a temp sandbox so it
+// can never clobber a real (possibly in-flight) run's results/receipts
+const RESULTS_ROOT = process.env.LOCAL_RESULTS_ROOT ?? ROOT;
+const OUT_DIR = join(RESULTS_ROOT, 'results-local');
 const PUBLISHED_FILES = [
   'apps/web/src/data/regression-eval-results.json',
   'apps/web/src/data/eval-results.json',
@@ -317,7 +320,12 @@ function runEval(
 
 // test hook: LOCAL_EVAL_CMD writes the result file itself (no model spend)
 function fakeRun(evalId: string, experiment: string): string {
-  const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`);
+  const resultPath = join(
+    RESULTS_ROOT,
+    'results',
+    experiment,
+    `${evalId}.json`
+  );
   mkdirSync(dirname(resultPath), { recursive: true });
   const res = spawnSync(process.env.LOCAL_EVAL_CMD as string, {
     shell: true,
@@ -449,8 +457,36 @@ async function cmdExperiments() {
   }
 }
 
+const SUITE_FILE: Record = {
+  regression: 'apps/web/src/data/regression-eval-results.json',
+  benchmark: 'apps/web/src/data/eval-results.json',
+};
+
+/** Expand --suite to every eval the published export carries for the experiment. */
+function expandSuite(suite: string, experiment: string): string[] {
+  const file = SUITE_FILE[suite];
+  if (!file)
+    fail(
+      `unknown suite: ${suite} (available: ${Object.keys(SUITE_FILE).join(', ')})`
+    );
+  const rows = loadPublishedFile(file)?.rows ?? [];
+  const ids = [
+    ...new Set(
+      rows.filter((r) => r.experiment === experiment).map((r) => r.eval)
+    ),
+  ].sort();
+  if (!ids.length)
+    fail(
+      `no ${suite} rows published for ${experiment} (published experiments: ${[...new Set(rows.map((r) => r.experiment))].join(', ')})`
+    );
+  console.log(
+    `suite ${suite} for ${experiment}: ${ids.length} evals (one model run each)\n  ${ids.join('\n  ')}`
+  );
+  return ids;
+}
+
 const RUN_USAGE =
-  'usage: pnpm local   [...] [--experiment ] [--runs N] [--mcp ] [--content-api ]';
+  'usage: pnpm local   [--experiment ] [--runs N] [--mcp ] [--content-api ]\n       pnpm local compare --suite  [same flags]   # every eval published for the experiment';
 
 async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
   const parsed = (() => {
@@ -460,6 +496,7 @@ async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
         options: {
           experiment: { type: 'string' },
           runs: { type: 'string' },
+          suite: { type: 'string' },
           mcp: { type: 'string' },
           'content-api': { type: 'string' },
         },
@@ -469,10 +506,20 @@ async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
       fail(`${err instanceof Error ? err.message : String(err)}\n${RUN_USAGE}`);
     }
   })();
-  const { values, positionals: evalIds } = parsed;
-  if (!evalIds.length) fail(RUN_USAGE);
+  const { values, positionals } = parsed;
   const experiment = values.experiment ?? DEFAULT_EXPERIMENT;
   validateExperiment(experiment);
+  let evalIds = positionals;
+  if (values.suite) {
+    if (mode !== 'compare')
+      fail(
+        '--suite expands from the published exports and only makes sense with compare'
+      );
+    if (evalIds.length) fail('pass either eval ids or --suite, not both');
+    if (!process.env.LOCAL_NO_FETCH) git(['fetch', '-q', 'origin', 'main']);
+    evalIds = expandSuite(values.suite, experiment);
+  }
+  if (!evalIds.length) fail(RUN_USAGE);
 
   const baselines =
     mode === 'compare'
diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts
index 63685c01..1e99289e 100644
--- a/apps/framework/scripts/smoke-local.ts
+++ b/apps/framework/scripts/smoke-local.ts
@@ -11,15 +11,21 @@ import { execFileSync, spawnSync } from 'node:child_process';
 import {
   existsSync,
   mkdirSync,
+  mkdtempSync,
   readFileSync,
   rmSync,
   writeFileSync,
 } from 'node:fs';
 import { dirname, join } from 'node:path';
+import { tmpdir } from 'node:os';
 import { fileURLToPath } from 'node:url';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = join(__dirname, '..', '..', '..');
+// every output lands in a disposable sandbox — never the checkout's real
+// results/ or results-local/ (an in-flight manual run may own those)
+const SANDBOX = mkdtempSync(join(tmpdir(), 'smoke-local-'));
+const OUT = join(SANDBOX, 'results-local');
 const EXPERIMENT = 'claude-code-sonnet-5';
 
 // a published, currently-existing eval id — resolved dynamically so the test
@@ -53,6 +59,7 @@ function local(args: string[], env: Record = {}) {
       env: {
         ...process.env,
         LOCAL_NO_FETCH: '1',
+        LOCAL_RESULTS_ROOT: SANDBOX,
         LOCAL_EVAL_CMD: FAKE,
         FORCE_COLOR: '0',
         ...env,
@@ -109,20 +116,14 @@ function ck(name: string, fn: () => void) {
   });
   ck('published receipt carries commit provenance', () => {
     const receipt = JSON.parse(
-      readFileSync(
-        join(ROOT, 'results-local', `${EVAL}.published.json`),
-        'utf8'
-      )
+      readFileSync(join(OUT, `${EVAL}.published.json`), 'utf8')
     );
     assert.match(receipt.publishedProvenance.commit, /^[0-9a-f]{40}$/);
     assert.match(receipt.publishedProvenance.parent, /^[0-9a-f]{40}$/);
   });
   ck('treatment receipt carries host provenance', () => {
     const receipt = JSON.parse(
-      readFileSync(
-        join(ROOT, 'results-local', `${EVAL}.treatment.json`),
-        'utf8'
-      )
+      readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8')
     );
     assert.match(receipt.provenance.host.sha, /^[0-9a-f]{40}$/);
     assert.equal(typeof receipt.provenance.host.dirtyFiles, 'number');
@@ -151,7 +152,7 @@ function ck(name: string, fn: () => void) {
 
 // --- mcp override: monorepo root resolves to the server package; unbuilt refused ---
 {
-  const fake = join(ROOT, 'results-local', '.smoke-mcp-checkout');
+  const fake = join(SANDBOX, '.smoke-mcp-checkout');
   const pkg = join(fake, 'packages', 'mcp-server-supabase');
   mkdirSync(join(pkg, 'dist', 'transports'), { recursive: true });
 
@@ -170,10 +171,7 @@ function ck(name: string, fn: () => void) {
   ck('monorepo root resolves to the server package dir', () => {
     assert.equal(built.status, 0);
     const receipt = JSON.parse(
-      readFileSync(
-        join(ROOT, 'results-local', `${EVAL}.treatment.json`),
-        'utf8'
-      )
+      readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8')
     );
     assert.match(
       receipt.provenance.mcpOverride.path,
@@ -183,6 +181,34 @@ function ck(name: string, fn: () => void) {
   rmSync(fake, { recursive: true, force: true });
 }
 
+// --- --suite: expands to the published set; guarded against misuse ---
+{
+  const r = local(['compare', '--suite', 'regression']);
+  ck('suite expands and runs every published eval', () => {
+    assert.equal(r.status, 0);
+    assert.match(r.out, /suite regression for claude-code-sonnet-5: \d+ evals/);
+    assert.ok(
+      (r.out.match(/=== local compare: /g) ?? []).length >= 2,
+      'expected multiple compare blocks'
+    );
+  });
+  const wrongMode = local(['run', '--suite', 'regression']);
+  ck('suite refused in run mode', () => {
+    assert.equal(wrongMode.status, 1);
+    assert.match(wrongMode.out, /only makes sense with compare/);
+  });
+  const both = local(['compare', EVAL, '--suite', 'regression']);
+  ck('suite plus ids refused', () => {
+    assert.equal(both.status, 1);
+    assert.match(both.out, /not both/);
+  });
+  const bogus = local(['compare', '--suite', 'nope']);
+  ck('unknown suite lists available', () => {
+    assert.equal(bogus.status, 1);
+    assert.match(bogus.out, /unknown suite: nope.*regression, benchmark/);
+  });
+}
+
 // --- judge-key gate: refused pre-spend, before any agent spawn ---
 {
   // needs an eval whose scorer really uses the judge; EVAL may not
@@ -206,9 +232,7 @@ function ck(name: string, fn: () => void) {
   });
 }
 
-// cleanup
-rmSync(join(ROOT, 'results-local', `${EVAL}.published.json`), { force: true });
-rmSync(join(ROOT, 'results-local', `${EVAL}.treatment.json`), { force: true });
-rmSync(join(ROOT, 'results', EXPERIMENT, `${EVAL}.json`), { force: true });
+// cleanup: everything lived in the sandbox
+rmSync(SANDBOX, { recursive: true, force: true });
 
 console.log(`smoke-local: ${passed} checks passed`);

From dd60687d36395ba0e3666ff5c4fdff4cedf24edc Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 12:19:39 +0200
Subject: [PATCH 12/24] refactor: load published exports once per invocation;
 script-file test fake
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Second strict-review pass. The incremental fix rounds left suite-mode
fetching origin/main twice (suite branch + inside resolveBaselines) and
parsing both multi-MB published exports twice per run, plus SUITE_FILE
duplicating PUBLISHED_FILES' literals. Now: PUBLISHED_EXPORTS is the one
suite->file map (values double as the load list), compare mode does
fetchMain() + loadPublished() exactly once, and resolveBaselines/
expandSuite are pure over the preloaded files (the fetch param is gone).

smoke-local's per-platform quoted FAKE one-liner pair becomes a script
file written into the sandbox — one string, no shell-quoting dialects.
15/15 checks; typecheck, biome clean; behavior verified unchanged.
---
 apps/framework/scripts/local.ts       | 71 ++++++++++++++++-----------
 apps/framework/scripts/smoke-local.ts | 25 ++++++++--
 2 files changed, 61 insertions(+), 35 deletions(-)

diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts
index e8f0c910..dbdfe7fb 100755
--- a/apps/framework/scripts/local.ts
+++ b/apps/framework/scripts/local.ts
@@ -49,10 +49,11 @@ const ROOT = resolve(__dirname, '..', '..', '..');
 // can never clobber a real (possibly in-flight) run's results/receipts
 const RESULTS_ROOT = process.env.LOCAL_RESULTS_ROOT ?? ROOT;
 const OUT_DIR = join(RESULTS_ROOT, 'results-local');
-const PUBLISHED_FILES = [
-  'apps/web/src/data/regression-eval-results.json',
-  'apps/web/src/data/eval-results.json',
-];
+// suite name -> published export file; the values double as the full load list
+const PUBLISHED_EXPORTS: Record = {
+  regression: 'apps/web/src/data/regression-eval-results.json',
+  benchmark: 'apps/web/src/data/eval-results.json',
+};
 const DEFAULT_EXPERIMENT = 'claude-code-sonnet-5';
 
 function fail(msg: string): never {
@@ -148,6 +149,25 @@ function loadPublishedFile(file: string): PublishedFile | undefined {
   return { file, rows, commit, parent, committedAt };
 }
 
+/** Fetch origin/main so the published exports are current; warn-and-continue offline. */
+function fetchMain() {
+  if (process.env.LOCAL_NO_FETCH) return;
+  try {
+    git(['fetch', '-q', 'origin', 'main']);
+  } catch {
+    console.error(
+      'warning: could not fetch origin/main — comparing against the local ref, which may be stale'
+    );
+  }
+}
+
+/** Load every published export once; callers share the result. */
+function loadPublished(): PublishedFile[] {
+  return Object.values(PUBLISHED_EXPORTS).flatMap(
+    (f) => loadPublishedFile(f) ?? []
+  );
+}
+
 /**
  * Freshest published row per requested eval for the experiment. Refuses
  * (pre-spend) when any requested eval has no published row, listing the
@@ -156,18 +176,8 @@ function loadPublishedFile(file: string): PublishedFile | undefined {
 function resolveBaselines(
   evalIds: string[],
   experiment: string,
-  fetch: boolean
+  files: PublishedFile[]
 ): Map {
-  if (fetch) {
-    try {
-      git(['fetch', '-q', 'origin', 'main']);
-    } catch {
-      console.error(
-        'warning: could not fetch origin/main — comparing against the local ref, which may be stale'
-      );
-    }
-  }
-  const files = PUBLISHED_FILES.flatMap((f) => loadPublishedFile(f) ?? []);
   const best = new Map();
   const failures: string[] = [];
   for (const id of evalIds) {
@@ -435,9 +445,7 @@ function runTreatment(
 
 async function cmdExperiments() {
   const published = new Set(
-    PUBLISHED_FILES.flatMap(
-      (f) => loadPublishedFile(f)?.rows.map((r) => r.experiment) ?? []
-    )
+    loadPublished().flatMap((f) => f.rows.map((r) => r.experiment))
   );
   console.log(
     `${'EXPERIMENT'.padEnd(36)} ${'AGENT'.padEnd(12)} ${'MODEL'.padEnd(22)} ${'EFFORT'.padEnd(8)} PUBLISHED`
@@ -457,19 +465,18 @@ async function cmdExperiments() {
   }
 }
 
-const SUITE_FILE: Record = {
-  regression: 'apps/web/src/data/regression-eval-results.json',
-  benchmark: 'apps/web/src/data/eval-results.json',
-};
-
 /** Expand --suite to every eval the published export carries for the experiment. */
-function expandSuite(suite: string, experiment: string): string[] {
-  const file = SUITE_FILE[suite];
+function expandSuite(
+  suite: string,
+  experiment: string,
+  files: PublishedFile[]
+): string[] {
+  const file = PUBLISHED_EXPORTS[suite];
   if (!file)
     fail(
-      `unknown suite: ${suite} (available: ${Object.keys(SUITE_FILE).join(', ')})`
+      `unknown suite: ${suite} (available: ${Object.keys(PUBLISHED_EXPORTS).join(', ')})`
     );
-  const rows = loadPublishedFile(file)?.rows ?? [];
+  const rows = files.filter((f) => f.file === file).flatMap((f) => f.rows);
   const ids = [
     ...new Set(
       rows.filter((r) => r.experiment === experiment).map((r) => r.eval)
@@ -509,6 +516,11 @@ async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
   const { values, positionals } = parsed;
   const experiment = values.experiment ?? DEFAULT_EXPERIMENT;
   validateExperiment(experiment);
+  let published: PublishedFile[] = [];
+  if (mode === 'compare') {
+    fetchMain();
+    published = loadPublished();
+  }
   let evalIds = positionals;
   if (values.suite) {
     if (mode !== 'compare')
@@ -516,14 +528,13 @@ async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
         '--suite expands from the published exports and only makes sense with compare'
       );
     if (evalIds.length) fail('pass either eval ids or --suite, not both');
-    if (!process.env.LOCAL_NO_FETCH) git(['fetch', '-q', 'origin', 'main']);
-    evalIds = expandSuite(values.suite, experiment);
+    evalIds = expandSuite(values.suite, experiment, published);
   }
   if (!evalIds.length) fail(RUN_USAGE);
 
   const baselines =
     mode === 'compare'
-      ? resolveBaselines(evalIds, experiment, !process.env.LOCAL_NO_FETCH)
+      ? resolveBaselines(evalIds, experiment, published)
       : new Map();
 
   validateEvals(evalIds);
diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts
index 1e99289e..b3e629f8 100644
--- a/apps/framework/scripts/smoke-local.ts
+++ b/apps/framework/scripts/smoke-local.ts
@@ -42,11 +42,26 @@ const EVAL = published.find(
 )?.eval;
 assert.ok(EVAL, 'no published eval with a local evals/ dir found');
 
-// LOCAL_EVAL_CMD contract: write a result JSON to $RES for eval $EVAL
-const FAKE =
-  process.platform === 'win32'
-    ? `node -e "require('fs').mkdirSync(require('path').dirname(process.env.RES),{recursive:true});require('fs').writeFileSync(process.env.RES,JSON.stringify({eval:process.env.EVAL,experiment:'${EXPERIMENT}',passed:true,checks:[{name:'x',passed:true}]}))"`
-    : `node -e 'require("fs").mkdirSync(require("path").dirname(process.env.RES),{recursive:true});require("fs").writeFileSync(process.env.RES,JSON.stringify({eval:process.env.EVAL,experiment:"${EXPERIMENT}",passed:true,checks:[{name:"x",passed:true}]}))'`;
+// LOCAL_EVAL_CMD contract: write a result JSON to $RES for eval $EVAL.
+// A script file sidesteps per-platform shell quoting entirely.
+const fakeScript = join(SANDBOX, 'fake-eval.cjs');
+writeFileSync(
+  fakeScript,
+  `const fs = require('node:fs');
+const path = require('node:path');
+fs.mkdirSync(path.dirname(process.env.RES), { recursive: true });
+fs.writeFileSync(
+  process.env.RES,
+  JSON.stringify({
+    eval: process.env.EVAL,
+    experiment: '${EXPERIMENT}',
+    passed: true,
+    checks: [{ name: 'x', passed: true }],
+  })
+);
+`
+);
+const FAKE = `${JSON.stringify(process.execPath)} ${JSON.stringify(fakeScript)}`;
 
 function local(args: string[], env: Record = {}) {
   const res = spawnSync(

From e9f7022f8de9f2f18372b08082eabf832d213e5f Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 12:46:17 +0200
Subject: [PATCH 13/24] feat: agent-key pre-spend gate + fixture pin-drift
 warning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two frictions from validating mcp#333 through the runner:

- Missing ANTHROPIC_API_KEY made the harness SKIP with exit 0, so the
  runner failed only at the no-result check. validateAgentKey refuses
  pre-spend — and distinguishes set-but-EMPTY (a stray `export KEY=`
  silently shadows .env because node --env-file never overrides an
  existing var; observed live) from genuinely unset.
- A local mcp build from a newer release line can call endpoints the
  platform-lite fixture (which tracks the MCP_SERVER_VERSION pin) does not
  serve: get_logs moved logs.all -> /logs in 0.9.0 while pin + fixture sat
  at 0.8.x, so every logs eval 404s against local builds with no hint why.
  resolveMcpServerPath now compares the build's package version to the pin
  and prints a judge-by-activation warning on mismatch.
---
 apps/framework/scripts/local.ts | 34 +++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts
index dbdfe7fb..6e49b200 100755
--- a/apps/framework/scripts/local.ts
+++ b/apps/framework/scripts/local.ts
@@ -34,6 +34,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
 import { parseArgs, type ParseArgsConfig } from 'node:util';
 import {
   getExperimentDisplayMetadata,
+  MCP_SERVER_VERSION,
   type ExperimentConfig,
 } from '@supabase-evals/core';
 import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown';
@@ -239,6 +240,23 @@ function validateExperiment(experiment: string) {
   }
 }
 
+/**
+ * The agent itself needs its provider key. Checked here (not just by the
+ * harness) because the harness SKIPs the experiment with exit 0 on missing
+ * credentials — the runner would only notice at the no-result check. A
+ * set-but-EMPTY var counts as missing (node --env-file does not override
+ * an existing env var, even an empty one, so a stray `export KEY=` in the
+ * shell silently shadows .env — observed live).
+ */
+function validateAgentKey() {
+  if (process.env.ANTHROPIC_API_KEY) return;
+  fail(
+    process.env.ANTHROPIC_API_KEY === undefined
+      ? 'ANTHROPIC_API_KEY not set — add it to .env at the repo root'
+      : 'ANTHROPIC_API_KEY is set but EMPTY in your shell, which shadows .env (node --env-file never overrides an existing var) — unset it or export a real value'
+  );
+}
+
 /**
  * Evals whose scorer uses the LLM judge grade the agent's output with an
  * OpenAI model — even when the agent under test is Claude. A missing grader
@@ -271,6 +289,21 @@ function resolveMcpServerPath(raw: string): string {
     fail(
       `no built server at ${p} (dist/transports/stdio.js missing) — build it first:\n  pnpm install && pnpm build   # in the mcp checkout (use \`mise exec --\` if corepack's pnpm mismatches)`
     );
+  // Fixture-drift heads-up: platform-lite tracks the pinned package version,
+  // and a local build from a newer line may call endpoints the fixture does
+  // not serve yet (observed: get_logs moved logs.all -> logs in 0.9.0 while
+  // the pin and fixture sat at 0.8.x). Warn, don't block.
+  try {
+    const local = JSON.parse(
+      readFileSync(join(p, 'package.json'), 'utf8')
+    ).version;
+    if (local && local !== MCP_SERVER_VERSION)
+      console.error(
+        `note: local mcp build is v${local}; the harness fixture (platform-lite) tracks the v${MCP_SERVER_VERSION} pin — endpoint drift is possible; judge by tool-call activation, not pass/fail alone`
+      );
+  } catch {
+    /* unversioned checkout: nothing to compare */
+  }
   return p;
 }
 
@@ -540,6 +573,7 @@ async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
   validateEvals(evalIds);
   // these gates are spend-relevant only for real runs; the test hook fakes them
   if (!process.env.LOCAL_EVAL_CMD) {
+    validateAgentKey();
     await validateSkills(experiment);
     validateJudgeKeys(evalIds);
   }

From 4626733e195db8cd472a714157a635c2dae85e09 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 12:46:54 +0200
Subject: [PATCH 14/24] docs: dependency-PR validation recipe in AGENTS.md
 (from the mcp#333 round)

---
 AGENTS.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/AGENTS.md b/AGENTS.md
index b6fbe330..c26661ba 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -53,6 +53,24 @@ arm's result commit + parent + age.
 - **No published baseline?** Use `pnpm local run` (custom evals included).
   For a before/after, run once before the edit and once after.
 
+## Validating a dependency PR (e.g. supabase/mcp)
+
+1. **Baseline-proof first**: build the dependency's MAIN and run the chosen
+   eval(s) against it before the PR build — version pins hide fixture drift
+   (platform-lite tracks the pinned `MCP_SERVER_VERSION`, not your local
+   build's line; the runner warns on version mismatch).
+2. Fixture or eval support living in an unmerged evals PR? Apply it into the
+   worktree as plain working-tree state: `gh pr diff  | git apply`.
+   Receipts record the dirty tree, so runs stay attributable.
+3. Run the PR build with `--mcp `; a main-FAIL -> PR-PASS flip with
+   everything else constant is a true two-arm comparison on the dependency
+   axis (stronger than the published screen).
+4. **Judge by tool-call activation, not pass/fail**: confirm the changed tool
+   was called, and unwrap `` envelopes in `toolCalls[]`
+   before reading results — errors hide inside them. Note that claude-code
+   records endpoints with an `mcp____` prefix; match with
+   `.endsWith('')`.
+
 ## Spend rules
 
 - Eval runs cost model tokens; `pnpm local docs seed` costs **~$0.12 OpenAI

From 419b5bce9dbca71a78c3b97fa44942dff36224a2 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 13:52:23 +0200
Subject: [PATCH 15/24] feat: gate --content-api on an mcp build that honours
 it

SUPABASE_CONTENT_API_URL support merged in supabase/mcp#343 but ships in no
release yet (newest v0.9.0; pin is 0.8.1). Ungated, `--content-api` with the
published package was a silent no-op: search_docs queried PRODUCTION docs while
collectProvenance still stamped contentApiUrl into the receipt, so a paid run
measured the wrong world and reported the right one.

Refuse pre-spend when --content-api arrives without --mcp, or when the supplied
build's dist does not read the env var. Shape check, so it also covers fake
runs. +3 smoke checks (both refusals and the honoured-build accept path).
---
 apps/framework/scripts/local.ts       | 27 ++++++++++++++++-
 apps/framework/scripts/smoke-local.ts | 42 +++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts
index 6e49b200..10ec4c61 100755
--- a/apps/framework/scripts/local.ts
+++ b/apps/framework/scripts/local.ts
@@ -307,6 +307,28 @@ function resolveMcpServerPath(raw: string): string {
   return p;
 }
 
+/**
+ * `--content-api` is only honoured by a local mcp build. The flag sets
+ * SUPABASE_CONTENT_API_URL, which the server reads to point `search_docs` at
+ * a local docs index — support merged in supabase/mcp#343 but shipped in NO
+ * release yet (newest is v0.9.0; the harness pin is older still). Ungated,
+ * the published package ignores the var: `search_docs` silently queries
+ * PRODUCTION docs while collectProvenance still stamps contentApiUrl into the
+ * receipt — a paid run that measures the wrong world and reports the right
+ * one. Shape check, so it runs for fake runs too (like resolveMcpServerPath).
+ */
+function validateContentApi(contentApi: string, mcpServerPath?: string) {
+  if (!mcpServerPath)
+    fail(
+      `--content-api needs --mcp : the published mcp server ignores SUPABASE_CONTENT_API_URL, so search_docs would query production docs while the receipt claims ${contentApi}\n  pass --mcp pointing at an mcp checkout on main, built (flag support merged in supabase/mcp#343, not yet released)`
+    );
+  const stdio = join(mcpServerPath, 'dist', 'transports', 'stdio.js');
+  if (!readFileSync(stdio, 'utf8').includes('SUPABASE_CONTENT_API_URL'))
+    fail(
+      `the mcp build at ${mcpServerPath} predates supabase/mcp#343 and ignores SUPABASE_CONTENT_API_URL — search_docs would query production docs, not ${contentApi}\n  update and rebuild the checkout: git pull && pnpm install && pnpm build`
+    );
+}
+
 /**
  * The experiment's declared skills must exist in this checkout, or the
  * treatment silently runs skill-less against a skills-enabled published
@@ -582,7 +604,10 @@ async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) {
   const mcpPath = values.mcp ? resolveMcpServerPath(values.mcp) : undefined;
   if (mcpPath) env.SUPABASE_MCP_SERVER_PATH = mcpPath;
   const contentApi = values['content-api'];
-  if (contentApi) env.SUPABASE_CONTENT_API_URL = contentApi;
+  if (contentApi) {
+    validateContentApi(contentApi, mcpPath);
+    env.SUPABASE_CONTENT_API_URL = contentApi;
+  }
 
   mkdirSync(OUT_DIR, { recursive: true });
   let exitCode = 0;
diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts
index b3e629f8..60a5bf05 100644
--- a/apps/framework/scripts/smoke-local.ts
+++ b/apps/framework/scripts/smoke-local.ts
@@ -193,6 +193,48 @@ function ck(name: string, fn: () => void) {
       /packages[/\\]mcp-server-supabase$/
     );
   });
+
+  // --- --content-api: refused unless a build that honours it is supplied ---
+  const noMcp = local(['run', EVAL, '--content-api', 'http://127.0.0.1:3001']);
+  ck('--content-api without --mcp refused pre-spend', () => {
+    assert.equal(noMcp.status, 1);
+    assert.match(noMcp.out, /--content-api needs --mcp/);
+    assert.match(noMcp.out, /production docs/);
+  });
+
+  // the fixture above is a bare stub, i.e. a build predating supabase/mcp#343
+  const staleBuild = local([
+    'run',
+    EVAL,
+    '--content-api',
+    'http://127.0.0.1:3001',
+    '--mcp',
+    fake,
+  ]);
+  ck('mcp build that ignores the env var refused pre-spend', () => {
+    assert.equal(staleBuild.status, 1);
+    assert.match(staleBuild.out, /predates supabase\/mcp#343/);
+  });
+
+  writeFileSync(
+    join(pkg, 'dist', 'transports', 'stdio.js'),
+    '// smoke fixture reading process.env.SUPABASE_CONTENT_API_URL\n'
+  );
+  const honoured = local([
+    'run',
+    EVAL,
+    '--content-api',
+    'http://127.0.0.1:3001',
+    '--mcp',
+    fake,
+  ]);
+  ck('build honouring the env var is accepted and recorded', () => {
+    assert.equal(honoured.status, 0);
+    const receipt = JSON.parse(
+      readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8')
+    );
+    assert.equal(receipt.provenance.contentApiUrl, 'http://127.0.0.1:3001');
+  });
   rmSync(fake, { recursive: true, force: true });
 }
 

From 7bc36f226765c916b4456702763936c59ea780df Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 14:04:18 +0200
Subject: [PATCH 16/24] fix: actually deliver --content-api to the mcp server

The gate added in 419b5bc stopped the silent-prod case but nothing delivered
the override: createConfig returns only {command,args}, and CLI agents spawn
that command INSIDE the sandbox container, which inherits nothing from the
harness process. So SUPABASE_CONTENT_API_URL never reached search_docs.

- createConfig now bakes --content-api-url into serverArgs, so rewriteLoopback
  maps 127.0.0.1 -> host.docker.internal for the containerised case (the same
  path --api-url already takes). Flag landed in supabase/mcp#343.
- The docs content API binds 0.0.0.0 (still advertised on 127.0.0.1), matching
  platform-lite in tools mode (run-eval.ts): on native Linux docker,
  host.docker.internal arrives on the bridge interface, which a loopback-only
  listener refuses.
- README/AGENTS: --content-api requires --mcp, and say why.

Proven hermetically (no model spend): harness config -> real mcp build (main,
#343) -> stdio -> local stub received both the schema load and the agent's
searchDocs query, canary surfaced in the tool result. Negative control with the
flag suppressed: zero stub requests, i.e. it went to production docs.
---
 AGENTS.md                                     |  6 +++-
 README.md                                     |  5 ++-
 .../scripts/docs/content-api-server.ts        |  7 +++-
 packages/core/src/index.ts                    | 11 +++++++
 packages/core/src/mcp-server.test.ts          | 33 +++++++++++++++++++
 5 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index c26661ba..421e7147 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -30,7 +30,11 @@ Per input:
 - **Docs page edited** (external supabase/supabase checkout):
   `pnpm local docs seed --yes` to re-embed (**~$0.12 OpenAI — see spend
   rules**), keep `pnpm local docs api` running in a separate terminal, then
-  `pnpm local compare  --content-api http://127.0.0.1:3001/docs/api/graphql`.
+  `pnpm local compare  --content-api http://127.0.0.1:3001/docs/api/graphql --mcp `.
+  `--content-api` also needs `--mcp`: the env var it sets is only read by an
+  mcp build carrying supabase/mcp#343 (merged, unreleased). With the published
+  package `search_docs` would query production docs while the receipt claimed
+  otherwise, so the runner refuses pre-spend.
 
 Receipts land in `results-local/` (git-ignored): treatment provenance (host
 SHA + dirty state, override git state) and, for `compare`, the published
diff --git a/README.md b/README.md
index cd0c0632..9e3883e0 100644
--- a/README.md
+++ b/README.md
@@ -101,7 +101,10 @@ pnpm local experiments                      # list experiments + published-basel
   pnpm local docs up --docs 
   pnpm local docs seed        # full embed via the docs app's pipeline (~$0.12 OpenAI; asks first)
   pnpm local docs api         # keep running in a separate terminal
-  pnpm local run  --content-api http://127.0.0.1:3001/docs/api/graphql
+  # --content-api needs a local mcp build too: SUPABASE_CONTENT_API_URL support
+  # is merged (supabase/mcp#343) but unreleased, so the published server ignores
+  # it and search_docs would silently hit production docs. Refused pre-spend.
+  pnpm local run  --content-api http://127.0.0.1:3001/docs/api/graphql --mcp 
   ```
 
 Every run writes a provenance receipt to `results-local/` (host SHA + dirty
diff --git a/apps/framework/scripts/docs/content-api-server.ts b/apps/framework/scripts/docs/content-api-server.ts
index 9e2ece16..d54a5a94 100644
--- a/apps/framework/scripts/docs/content-api-server.ts
+++ b/apps/framework/scripts/docs/content-api-server.ts
@@ -61,6 +61,11 @@ createServer(async (incoming, outgoing) => {
     Object.fromEntries(response.headers.entries())
   );
   outgoing.end(Buffer.from(await response.arrayBuffer()));
-}).listen(port, '127.0.0.1', () => {
+  // Bind every interface, advertise loopback — same rule as platform-lite in
+  // tools mode (see run-eval.ts: sandboxed CLI agents run their MCP servers
+  // INSIDE the container and reach host-side services via
+  // host.docker.internal, which arrives on the host's bridge interface, not
+  // loopback; a 127.0.0.1-only listener refuses those connections).
+}).listen(port, '0.0.0.0', () => {
   console.log(`Docs content API: http://127.0.0.1:${port}/docs/api/graphql`);
 });
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index e25fc37b..20d21ac2 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -937,6 +937,17 @@ export function supabaseMcpServer(
       // docs-only server runs standalone with no `--api-url`.
       if (apiUrl) serverArgs.push('--api-url', apiUrl);
 
+      // Docs override: point `search_docs` at a local content API instead of
+      // the public docs GraphQL. Baked into args rather than left to the parent
+      // environment because CLI agents spawn this command INSIDE the sandbox
+      // container, which inherits nothing from the harness process — and
+      // `rewriteLoopback` then maps 127.0.0.1 -> host.docker.internal so the
+      // host-side API is actually reachable from in there. Flag support landed
+      // in supabase/mcp#343 (unreleased), so this needs a local build; `pnpm
+      // local` refuses --content-api without one.
+      const contentApiUrl = process.env.SUPABASE_CONTENT_API_URL;
+      if (contentApiUrl) serverArgs.push('--content-api-url', contentApiUrl);
+
       const local = resolveLocalMcpServer();
       if (local) {
         // `node`, not process.execPath: CLI agents run this command INSIDE the
diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts
index bd3f81c3..9f30397a 100644
--- a/packages/core/src/mcp-server.test.ts
+++ b/packages/core/src/mcp-server.test.ts
@@ -18,6 +18,7 @@ import {
 } from 'node:fs';
 import { join, relative } from 'node:path';
 import { tmpdir } from 'node:os';
+import { rewriteLoopback } from './agents/shared.js';
 import {
   MCP_SERVER_VERSION,
   supabaseMcpServer,
@@ -25,8 +26,11 @@ import {
 } from './index.js';
 
 // Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test.
+// SUPABASE_CONTENT_API_URL is cleared too: it now adds server args, so an
+// ambient value would leak into every assertion below.
 function clearEnv() {
   vi.stubEnv('SUPABASE_MCP_SERVER_PATH', undefined);
+  vi.stubEnv('SUPABASE_CONTENT_API_URL', undefined);
 }
 
 // A real on-disk build layout: the override path is existence-checked, so the
@@ -119,6 +123,35 @@ describe('supabaseMcpServer().createConfig', () => {
       rmSync(linkDir, { recursive: true, force: true });
     }
   });
+
+  it('passes SUPABASE_CONTENT_API_URL as --content-api-url so it survives into the sandbox', async () => {
+    clearEnv();
+    vi.stubEnv(
+      'SUPABASE_CONTENT_API_URL',
+      'http://127.0.0.1:3001/docs/api/graphql'
+    );
+    const { config } = await supabaseMcpServer().createConfig({});
+    // In args, not env: a CLI agent spawns this inside the container, which
+    // inherits nothing from the harness process.
+    const flag = config.args.indexOf('--content-api-url');
+    expect(flag).toBeGreaterThan(-1);
+    expect(config.args[flag + 1]).toBe(
+      'http://127.0.0.1:3001/docs/api/graphql'
+    );
+
+    // ...and being in args is what lets the container reach the host-side API.
+    const rewritten = rewriteLoopback({ supabase: config });
+    expect(rewritten.supabase.args).toContain(
+      'http://host.docker.internal:3001/docs/api/graphql'
+    );
+  });
+
+  it('omits the flag when no local docs API is configured', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_CONTENT_API_URL', undefined);
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.args).not.toContain('--content-api-url');
+  });
 });
 
 describe('supabaseMcpServerMounts', () => {

From fe0d7c81ea19d1a59c8d21edbb3366fa964ef851 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 14:22:32 +0200
Subject: [PATCH 17/24] fix: satisfy the docs pipeline's MISC env gate in docs
 seed

generate-embeddings.ts hard-requires NEXT_PUBLIC_MISC_URL and
NEXT_PUBLIC_MISC_ANON_KEY before doing any work (they name the hosted "misc"
project, read by sources/partner-integrations.ts). docs seed passed only the
NEXT_PUBLIC_SUPABASE_* pair, so a seed against current supabase/supabase main
aborted at the gate before embedding anything.

Found running the loop end to end against a real checkout. NOT sufficient on its
own: the same seed then fails on prod-only sources (lint-warnings and github
discussions both hard-require DOCS_GITHUB_APP_* with no token fallback), which is
what the old eval-workspace carried a local-only skip patch for. Upstreaming a
skip flag remains the gate for a vanilla full seed.
---
 apps/framework/scripts/local-docs.ts | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/apps/framework/scripts/local-docs.ts b/apps/framework/scripts/local-docs.ts
index 58b6f9ab..0db7d4c1 100644
--- a/apps/framework/scripts/local-docs.ts
+++ b/apps/framework/scripts/local-docs.ts
@@ -191,6 +191,14 @@ async function cmdSeed(opts: DocsOptions) {
       NEXT_PUBLIC_SUPABASE_URL: env.API_URL,
       NEXT_PUBLIC_SUPABASE_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY,
       SUPABASE_SECRET_KEY: env.SECRET_KEY ?? env.SERVICE_ROLE_KEY,
+      // generate-embeddings.ts hard-requires these two before doing any work.
+      // It builds its own client from NEXT_PUBLIC_SUPABASE_URL +
+      // SUPABASE_SECRET_KEY, but sources/partner-integrations.ts reads the MISC
+      // pair to pull partner data from the hosted "misc" project. Pointed at the
+      // local stack they clear the gate; the partner source then finds no such
+      // tables, which is the right trade for a local docs index.
+      NEXT_PUBLIC_MISC_URL: env.API_URL,
+      NEXT_PUBLIC_MISC_ANON_KEY: env.PUBLISHABLE_KEY ?? env.ANON_KEY,
       OPENAI_API_KEY: process.env.OPENAI_API_KEY,
       NODE_ENV: 'development',
     },

From fb58aa1045b11562b3a027e1837e04ccc07c122d Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 14:24:47 +0200
Subject: [PATCH 18/24] docs: state the docs-seed blocker instead of
 advertising the loop as working

The docs leg is not usable from a vanilla supabase/supabase checkout: the
embedding pipeline fails closed on lint-warnings + github-discussion, which
require a GitHub App (DOCS_GITHUB_APP_*, no token fallback). Verified by running
it. The run/compare half IS verified against an externally seeded index, so say
exactly that rather than implying the whole loop works.

Also record the retrieval-vs-answer scoring rule: tools mode exposes
WebSearch/WebFetch, and a docs edit contradicting the live page can be detected
and rejected by the agent as prompt injection (observed, not theorised).
---
 AGENTS.md | 8 ++++++++
 README.md | 9 +++++++++
 2 files changed, 17 insertions(+)

diff --git a/AGENTS.md b/AGENTS.md
index 421e7147..d45ed061 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -35,6 +35,14 @@ Per input:
   mcp build carrying supabase/mcp#343 (merged, unreleased). With the published
   package `search_docs` would query production docs while the receipt claimed
   otherwise, so the runner refuses pre-spend.
+  **`docs seed` currently fails against a vanilla docs checkout**: the pipeline
+  requires `DOCS_GITHUB_APP_*` for its lint-warnings and github-discussion
+  sources and fails closed without them. Do not burn spend retrying it; the leg
+  needs an index seeded another way until a source-skip flag lands upstream.
+  Score docs evals on retrieval (`docs.calls`, canary content coming back out of
+  `search_docs`), not on the answer text: tools mode also exposes
+  `WebSearch`/`WebFetch`, and an edit that contradicts the live page invites the
+  agent to fetch production and reject the local content as injection (observed).
 
 Receipts land in `results-local/` (git-ignored): treatment provenance (host
 SHA + dirty state, override git state) and, for `compare`, the published
diff --git a/README.md b/README.md
index 9e3883e0..0d62cab2 100644
--- a/README.md
+++ b/README.md
@@ -107,6 +107,15 @@ pnpm local experiments                      # list experiments + published-basel
   pnpm local run  --content-api http://127.0.0.1:3001/docs/api/graphql --mcp 
   ```
 
+  **Known limitation — `docs seed` does not complete against a vanilla docs
+  checkout yet.** The docs app's embedding pipeline fails closed on two
+  prod-only sources: `lint-warnings-guide` and `github-discussion` both require
+  `DOCS_GITHUB_APP_{ID,INSTALLATION_ID,PRIVATE_KEY}` (a GitHub App, no token
+  fallback), so the seed aborts before embedding. Until a source-skip flag lands
+  upstream in `supabase/supabase`, this leg needs an index seeded by some other
+  means; the `run`/`compare` side of the loop is verified against one (a real
+  agent's `search_docs` returned local-index-only content).
+
 Every run writes a provenance receipt to `results-local/` (host SHA + dirty
 state, override paths and their git state). `compare` records the published
 arm's result commit, parent, and age — and a pass/fail flip against published

From 7c0bed3cb508758ac1306654c6b96b2045b79142 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 14:32:33 +0200
Subject: [PATCH 19/24] docs: correct docs-seed blocker description

Only fetchLintWarningsGuideSources() blocks: it is awaited unconditionally and
throws without the GitHub App creds, and one shared Promise.all makes that fatal.
github-discussion has zero callers in the pipeline, so naming it as a blocker was
wrong. Also state why the skip belongs upstream (this runner takes an arbitrary
checkout; per-user patches are not distributable) and that a skip must preserve
the skipped source's rows, since the purge deletes every page the run did not
stamp.
---
 AGENTS.md |  8 +++++---
 README.md | 19 ++++++++++++-------
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index d45ed061..7c43401e 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -36,9 +36,11 @@ Per input:
   package `search_docs` would query production docs while the receipt claimed
   otherwise, so the runner refuses pre-spend.
   **`docs seed` currently fails against a vanilla docs checkout**: the pipeline
-  requires `DOCS_GITHUB_APP_*` for its lint-warnings and github-discussion
-  sources and fails closed without them. Do not burn spend retrying it; the leg
-  needs an index seeded another way until a source-skip flag lands upstream.
+  unconditionally loads its lint-warnings source, which needs a GitHub App
+  (`DOCS_GITHUB_APP_*`, no token fallback), and one `Promise.all` makes that
+  fatal. It aborts before embedding, so a retry costs nothing but achieves
+  nothing — don't loop on it. The leg needs an index seeded another way until a
+  skip flag lands upstream.
   Score docs evals on retrieval (`docs.calls`, canary content coming back out of
   `search_docs`), not on the answer text: tools mode also exposes
   `WebSearch`/`WebFetch`, and an edit that contradicts the live page invites the
diff --git a/README.md b/README.md
index 0d62cab2..bfe69fa3 100644
--- a/README.md
+++ b/README.md
@@ -108,13 +108,18 @@ pnpm local experiments                      # list experiments + published-basel
   ```
 
   **Known limitation — `docs seed` does not complete against a vanilla docs
-  checkout yet.** The docs app's embedding pipeline fails closed on two
-  prod-only sources: `lint-warnings-guide` and `github-discussion` both require
-  `DOCS_GITHUB_APP_{ID,INSTALLATION_ID,PRIVATE_KEY}` (a GitHub App, no token
-  fallback), so the seed aborts before embedding. Until a source-skip flag lands
-  upstream in `supabase/supabase`, this leg needs an index seeded by some other
-  means; the `run`/`compare` side of the loop is verified against one (a real
-  agent's `search_docs` returned local-index-only content).
+  checkout yet.** `fetchAllSources()` unconditionally awaits
+  `fetchLintWarningsGuideSources()`, whose loader throws without
+  `DOCS_GITHUB_APP_{ID,INSTALLATION_ID,PRIVATE_KEY}` (a GitHub App —
+  `createAppAuth`, no token fallback). All sources go through one `Promise.all`,
+  so that single rejection aborts the run — before any embedding, so it costs
+  nothing. A skip flag has to be upstream because this runner takes an arbitrary
+  checkout: patching yours is what the previous iteration did, and it is not
+  something we can ask every user (or CI) to carry. Any skip must also keep the
+  skipped source's existing rows, since the pipeline deletes every page whose
+  `version` was not stamped by the current run. Until then this leg needs an
+  index seeded another way; the `run`/`compare` side is verified against one (a
+  real agent's `search_docs` returned local-index-only content).
 
 Every run writes a provenance receipt to `results-local/` (host SHA + dirty
 state, override paths and their git state). `compare` records the published

From ab06e12c2311930087e36c5053f37134cc08ff47 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 14:34:40 +0200
Subject: [PATCH 20/24] docs: retract the purge-coupling claim about the old
 patches
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Verified both: lint-warnings-skip only returns [], and fail-closed gates the purge
on counted preparation/page/checksum failures. An intentional skip increments none
of them, so the old setup did NOT protect that source's rows — the purge still ran.
It never bit because a fresh index had no such rows.

Minimal upstream fix is the opt-in skip alone. Skip-aware purge narrowing is a
separate, newly identified retention requirement for re-running against an index
that already holds those rows.
---
 README.md | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index bfe69fa3..6ce4e9f5 100644
--- a/README.md
+++ b/README.md
@@ -113,13 +113,19 @@ pnpm local experiments                      # list experiments + published-basel
   `DOCS_GITHUB_APP_{ID,INSTALLATION_ID,PRIVATE_KEY}` (a GitHub App —
   `createAppAuth`, no token fallback). All sources go through one `Promise.all`,
   so that single rejection aborts the run — before any embedding, so it costs
-  nothing. A skip flag has to be upstream because this runner takes an arbitrary
-  checkout: patching yours is what the previous iteration did, and it is not
-  something we can ask every user (or CI) to carry. Any skip must also keep the
-  skipped source's existing rows, since the pipeline deletes every page whose
-  `version` was not stamped by the current run. Until then this leg needs an
-  index seeded another way; the `run`/`compare` side is verified against one (a
-  real agent's `search_docs` returned local-index-only content).
+  nothing. The skip belongs upstream for one reason only: this runner takes an
+  arbitrary vanilla checkout, so a patch on yours (what the previous iteration
+  carried) is not something every user or CI job can be asked to hold. The
+  minimal fix is just the opt-in skip — on a fresh local index the global
+  `version` purge has no rows of that source to remove. Separately, and newly
+  identified here: re-running against an index that ALREADY holds lint-warning
+  rows would delete them, because the purge removes every page the current run
+  did not stamp and a skipped source stamps nothing (the previous iteration's
+  fail-closed patch does not cover this — it only blocks the purge on counted
+  failures, and an intentional skip counts as success). Until the skip lands
+  this leg needs an index seeded another way; the `run`/`compare` side is
+  verified against one (a real agent's `search_docs` returned
+  local-index-only content).
 
 Every run writes a provenance receipt to `results-local/` (host SHA + dirty
 state, override paths and their git state). `compare` records the published

From 688517fa3f35524805a01d0a4c414e2233aae5c9 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 14:42:57 +0200
Subject: [PATCH 21/24] docs: prefer an auth ladder over a skip for the
 lint-warnings blocker

supabase/splinter is public and the loader makes 30 API calls, which fits the
unauthenticated 60/hr budget for a single run; a PAT or gh auth token gives 5,000.
So the gate is a rate-limit guard, not access control, and the better upstream fix
is App creds -> GITHUB_TOKEN/GH_TOKEN -> unauthenticated. That keeps the 29 Database
Advisor pages in the local index (no missing corpus, no purge-retention question)
and leaves production on the first rung.
---
 README.md | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 6ce4e9f5..59a8c888 100644
--- a/README.md
+++ b/README.md
@@ -113,19 +113,28 @@ pnpm local experiments                      # list experiments + published-basel
   `DOCS_GITHUB_APP_{ID,INSTALLATION_ID,PRIVATE_KEY}` (a GitHub App —
   `createAppAuth`, no token fallback). All sources go through one `Promise.all`,
   so that single rejection aborts the run — before any embedding, so it costs
-  nothing. The skip belongs upstream for one reason only: this runner takes an
-  arbitrary vanilla checkout, so a patch on yours (what the previous iteration
-  carried) is not something every user or CI job can be asked to hold. The
-  minimal fix is just the opt-in skip — on a fresh local index the global
-  `version` purge has no rows of that source to remove. Separately, and newly
-  identified here: re-running against an index that ALREADY holds lint-warning
-  rows would delete them, because the purge removes every page the current run
-  did not stamp and a skipped source stamps nothing (the previous iteration's
-  fail-closed patch does not cover this — it only blocks the purge on counted
-  failures, and an intentional skip counts as success). Until the skip lands
-  this leg needs an index seeded another way; the `run`/`compare` side is
-  verified against one (a real agent's `search_docs` returned
-  local-index-only content).
+  nothing. The blocker is a rate-limit guard, not an access one: `supabase/splinter`
+  is public, and the loader's 30 calls (1 directory listing + 29 lint files) fit
+  inside even the unauthenticated 60/hr budget for a single run. So the cleanest
+  upstream fix is an auth ladder in that loader — App creds, else
+  `GITHUB_TOKEN`/`GH_TOKEN` (5,000/hr, and `gh auth token` is already on most dev
+  machines), else unauthenticated. That keeps the advisor pages IN the local index
+  rather than skipping them, so there is no missing-corpus or purge-retention
+  question for this source at all, and production still takes the first rung.
+  An opt-in skip flag remains the right tool only for sources no contributor can
+  reach (partner integrations, via the hosted `misc` project). Either way it has
+  to land upstream: this runner takes an arbitrary vanilla checkout, so a patch on
+  yours (what the previous iteration carried) is not something every user or CI
+  job can be asked to hold. Until then this leg needs an index seeded another way;
+  the `run`/`compare` side is verified against one (a real agent's `search_docs`
+  returned local-index-only content).
+
+  If a skip is chosen instead, note a second requirement identified here and NOT
+  solved by the previous iteration's patches: re-running a skip against an index
+  that already holds those rows deletes them, because the purge removes every page
+  the run did not stamp and a skipped source stamps nothing (the fail-closed patch
+  only gates the purge on counted failures, and an intentional skip counts as
+  success).
 
 Every run writes a provenance receipt to `results-local/` (host SHA + dirty
 state, override paths and their git state). `compare` records the published

From 8ecd07b80b73337c2485802a877fa0396a55bc44 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 14:48:39 +0200
Subject: [PATCH 22/24] docs: cite the upstream rationale for the lint-warnings
 auth gate

#43015 documents it explicitly: unauthenticated calls caused flaky failures on
shared CI runners (failing job linked). And #44274 deliberately migrated GitHub
content fetches TO the authenticated Octokit client after raw.githubusercontent
failed in a preview, so proposing raw URLs would re-litigate that.

Revised recommendation: keep the authenticated client, add one PAT rung
(GITHUB_TOKEN/GH_TOKEN), no unauthenticated rung.
---
 README.md | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 59a8c888..142c752c 100644
--- a/README.md
+++ b/README.md
@@ -113,14 +113,24 @@ pnpm local experiments                      # list experiments + published-basel
   `DOCS_GITHUB_APP_{ID,INSTALLATION_ID,PRIVATE_KEY}` (a GitHub App —
   `createAppAuth`, no token fallback). All sources go through one `Promise.all`,
   so that single rejection aborts the run — before any embedding, so it costs
-  nothing. The blocker is a rate-limit guard, not an access one: `supabase/splinter`
-  is public, and the loader's 30 calls (1 directory listing + 29 lint files) fit
-  inside even the unauthenticated 60/hr budget for a single run. So the cleanest
-  upstream fix is an auth ladder in that loader — App creds, else
-  `GITHUB_TOKEN`/`GH_TOKEN` (5,000/hr, and `gh auth token` is already on most dev
-  machines), else unauthenticated. That keeps the advisor pages IN the local index
-  rather than skipping them, so there is no missing-corpus or purge-retention
-  question for this source at all, and production still takes the first rung.
+  nothing. GitHub imposes the limit (60/hr per IP unauthenticated); the loader
+  responded by hard-requiring App auth, and the reasoning is on record in
+  [supabase/supabase#43015](https://github.com/supabase/supabase/pull/43015):
+  unauthenticated calls were "causing flaky failures on shared CI runners", with
+  the failing job linked. Note this is not an access problem — `supabase/splinter`
+  is public and the loader makes only 30 calls (1 listing + 29 lint files).
+  Do NOT "fix" it by going back to `raw.githubusercontent.com`:
+  [#44274](https://github.com/supabase/supabase/pull/44274) deliberately migrated
+  those calls the other way, so that "all network calls for external content
+  coming from GitHub are done using our authenticated Octokit client", after raw
+  fetches failed in a preview.
+  The upstream change compatible with both: keep the authenticated Octokit client
+  and add ONE auth rung — App creds, else a `GITHUB_TOKEN`/`GH_TOKEN` PAT (still
+  authenticated, still 5,000/hr, so #43015's flakiness stays fixed), else fail
+  with a message naming the export. `gh auth token` supplies the value but is not
+  exported for you. That keeps the 29 advisor pages IN a local index instead of
+  skipping them, so no missing-corpus or purge-retention question arises for this
+  source, and production keeps taking the first rung unchanged.
   An opt-in skip flag remains the right tool only for sources no contributor can
   reach (partner integrations, via the hosted `misc` project). Either way it has
   to land upstream: this runner takes an arbitrary vanilla checkout, so a patch on

From 80deaa9b24162d1450ebd3cb308461592a097fd5 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Mon, 27 Jul 2026 15:48:56 +0200
Subject: [PATCH 23/24] docs: point the docs-seed limitation at
 supabase/supabase#48364

The block described the auth ladder as a proposal; it is now an open PR. Replaces
the design discussion with the actual requirement (a --docs checkout containing
#48364, or that branch checked out until it merges) and records what the
end-to-end run showed: the seed completes, and a tools-mode eval's search_docs
returns content that exists only in the local index.

Keeps the two upstream rough edges visible rather than implying the seed is
healthy: it exits 0 while silently failing 22 /reference/{javascript,dart} pages
whose sections exceed the 8192-token embedding limit, and a local index has no
partner-integration pages because that source reads the hosted misc project.

Also drops 'PAT': the rung accepts any token Octokit takes, including a gh OAuth
token or an Actions installation token.
---
 README.md | 56 ++++++++++++++++++-------------------------------------
 1 file changed, 18 insertions(+), 38 deletions(-)

diff --git a/README.md b/README.md
index 142c752c..681de0f6 100644
--- a/README.md
+++ b/README.md
@@ -107,44 +107,24 @@ pnpm local experiments                      # list experiments + published-basel
   pnpm local run  --content-api http://127.0.0.1:3001/docs/api/graphql --mcp 
   ```
 
-  **Known limitation — `docs seed` does not complete against a vanilla docs
-  checkout yet.** `fetchAllSources()` unconditionally awaits
-  `fetchLintWarningsGuideSources()`, whose loader throws without
-  `DOCS_GITHUB_APP_{ID,INSTALLATION_ID,PRIVATE_KEY}` (a GitHub App —
-  `createAppAuth`, no token fallback). All sources go through one `Promise.all`,
-  so that single rejection aborts the run — before any embedding, so it costs
-  nothing. GitHub imposes the limit (60/hr per IP unauthenticated); the loader
-  responded by hard-requiring App auth, and the reasoning is on record in
-  [supabase/supabase#43015](https://github.com/supabase/supabase/pull/43015):
-  unauthenticated calls were "causing flaky failures on shared CI runners", with
-  the failing job linked. Note this is not an access problem — `supabase/splinter`
-  is public and the loader makes only 30 calls (1 listing + 29 lint files).
-  Do NOT "fix" it by going back to `raw.githubusercontent.com`:
-  [#44274](https://github.com/supabase/supabase/pull/44274) deliberately migrated
-  those calls the other way, so that "all network calls for external content
-  coming from GitHub are done using our authenticated Octokit client", after raw
-  fetches failed in a preview.
-  The upstream change compatible with both: keep the authenticated Octokit client
-  and add ONE auth rung — App creds, else a `GITHUB_TOKEN`/`GH_TOKEN` PAT (still
-  authenticated, still 5,000/hr, so #43015's flakiness stays fixed), else fail
-  with a message naming the export. `gh auth token` supplies the value but is not
-  exported for you. That keeps the 29 advisor pages IN a local index instead of
-  skipping them, so no missing-corpus or purge-retention question arises for this
-  source, and production keeps taking the first rung unchanged.
-  An opt-in skip flag remains the right tool only for sources no contributor can
-  reach (partner integrations, via the hosted `misc` project). Either way it has
-  to land upstream: this runner takes an arbitrary vanilla checkout, so a patch on
-  yours (what the previous iteration carried) is not something every user or CI
-  job can be asked to hold. Until then this leg needs an index seeded another way;
-  the `run`/`compare` side is verified against one (a real agent's `search_docs`
-  returned local-index-only content).
-
-  If a skip is chosen instead, note a second requirement identified here and NOT
-  solved by the previous iteration's patches: re-running a skip against an index
-  that already holds those rows deletes them, because the purge removes every page
-  the run did not stamp and a skipped source stamps nothing (the fail-closed patch
-  only gates the purge on counted failures, and an intentional skip counts as
-  success).
+  **Known limitation — `docs seed` needs a docs checkout containing
+  [supabase/supabase#48364](https://github.com/supabase/supabase/pull/48364).**
+  Without it, `fetchAllSources()` unconditionally awaits the lint warnings source,
+  whose loader requires the docs GitHub App, and one shared `Promise.all` turns
+  that into a full abort before any embedding (so it costs nothing). That PR adds
+  a token rung below the App, `GH_TOKEN` then `GITHUB_TOKEN`, which is all a
+  contributor needs: `export GH_TOKEN=$(gh auth token)`. Until it merges, check
+  that branch out in the checkout you pass to `--docs`.
+
+  Verified end to end against a checkout carrying it, with the `NEXT_PUBLIC_MISC_*`
+  wiring `docs seed` supplies: the seed completes (1901 sources, 7890 sections) and
+  a tools-mode eval's `search_docs` returns content that exists only in the local
+  index. Two rough edges to expect, both upstream: the seed exits 0 while silently
+  failing 22 `/reference/{javascript,dart}` pages whose sections exceed the
+  embedding model's 8192-token limit, and a local index has no partner-integration
+  pages, since that source reads the hosted misc project. Neither blocked the
+  tested guide-page eval, but an eval whose answer lives in those reference pages
+  would find them missing from the index.
 
 Every run writes a provenance receipt to `results-local/` (host SHA + dirty
 state, override paths and their git state). `compare` records the published

From be6695147ce9621130ea34de9c4e32e267976eb5 Mon Sep 17 00:00:00 2001
From: Barry Roodt 
Date: Tue, 28 Jul 2026 10:19:41 +0200
Subject: [PATCH 24/24] chore: quiet the supabase CLI noise in pnpm local docs

`docs up` streamed the CLI's own output, which meant every run printed the
stack's ANON_KEY, PUBLISHABLE_KEY, SERVICE_ROLE_KEY, SECRET_KEY, and JWT_SECRET
as JSON, plus the workdir line, a config deprecation warning, the stopped-service
list, and the update-notifier nag. Our own one-line status was the last thing in
a 20-line wall, and the keys are not something to leave on a screen recording.

run() gains a quiet option that buffers and replays on failure, used for the
three supabase/docker calls. capture() now swallows stderr for the same reason
(execFileSync attaches output to the error, so failures still surface).

docs up now reports the API URL it actually read back from the stack, which also
means the status line only prints once the stack answers. docs down says where
the seeded index went, since stopping keeps the volume.

  starting the docs stack (project evals-local-docs)...
  docs stack up on http://127.0.0.1:44321; next: pnpm local docs seed
---
 apps/framework/scripts/local-docs.ts | 77 +++++++++++++++++++++-------
 1 file changed, 58 insertions(+), 19 deletions(-)

diff --git a/apps/framework/scripts/local-docs.ts b/apps/framework/scripts/local-docs.ts
index 0db7d4c1..7dcb99af 100644
--- a/apps/framework/scripts/local-docs.ts
+++ b/apps/framework/scripts/local-docs.ts
@@ -60,7 +60,21 @@ function fail(msg: string): never {
   process.exit(1);
 }
 
-/** Run a command, streaming output; fails loudly on nonzero exit. */
+/**
+ * Run a command, streaming output; fails loudly on nonzero exit.
+ *
+ * `quiet` buffers instead of streaming, because the supabase CLI reports the
+ * stack's ANON_KEY, PUBLISHABLE_KEY, SERVICE_ROLE_KEY, SECRET_KEY, and
+ * JWT_SECRET on every start, plus an update-notifier nag. That buries our own
+ * one-line status, and the keys are not something to leave on a screen
+ * recording.
+ *
+ * On failure it replays stderr only. Measured against `supabase start`: the key
+ * report goes to stdout (3 key lines there, 0 on stderr), while stderr carries
+ * the diagnostics you actually want (workdir, config warnings, per-service
+ * status). Dropping stdout is a stream boundary rather than a pattern match, so
+ * there is no redaction regex to keep in step with the CLI's output shapes.
+ */
 function run(
   cmd: string,
   args: string[],
@@ -68,21 +82,36 @@ function run(
     cwd?: string;
     env?: Record;
     shim?: boolean;
+    quiet?: boolean;
   } = {}
 ) {
   const res = spawnSync(cmd, args, {
-    stdio: 'inherit',
+    stdio: opts.quiet ? 'pipe' : 'inherit',
+    encoding: opts.quiet ? 'utf8' : undefined,
     cwd: opts.cwd ?? ROOT,
     env: opts.env ? { ...process.env, ...opts.env } : process.env,
     // .cmd shims (corepack, .bin/tsx) need a shell on Windows
     shell: opts.shim ? onWindows : false,
   });
-  if (res.status !== 0)
+  if (res.status !== 0) {
+    if (opts.quiet && res.stderr) process.stderr.write(res.stderr);
     fail(`${cmd} ${args.join(' ')} failed (exit ${res.status})`);
+  }
 }
 
+/**
+ * Capture stdout. stderr is swallowed rather than inherited: the supabase CLI
+ * writes its workdir line, deprecation warnings, stopped-service list, and
+ * update-notifier nag there on every invocation, and this runs inside helpers
+ * whose own output is one line. On failure execFileSync throws with the output
+ * attached, so nothing is lost when it matters.
+ */
 function capture(cmd: string, args: string[]): string {
-  return execFileSync(cmd, args, { cwd: ROOT, maxBuffer: 1 << 24 }).toString();
+  return execFileSync(cmd, args, {
+    cwd: ROOT,
+    maxBuffer: 1 << 24,
+    stdio: ['ignore', 'pipe', 'pipe'],
+  }).toString();
 }
 
 function docsPath(docs: string | undefined): string {
@@ -142,23 +171,30 @@ function cmdUp(opts: DocsOptions) {
   }
   writeFileSync(join(OVERLAY, 'docs-path.txt'), `${docs}\n`);
 
-  run('supabase', ['start', '--workdir', OVERLAY, '-x', STACK_EXCLUDES]);
+  console.log(`starting the docs stack (project ${PROJECT_ID})...`);
+  run('supabase', ['start', '--workdir', OVERLAY, '-x', STACK_EXCLUDES], {
+    quiet: true,
+  });
   // Upstream page migrations grant service_role no CRUD on the content
   // tables; the embedder authenticates as service_role and needs it.
-  run('docker', [
-    'exec',
-    DB_CONTAINER,
-    'psql',
-    '-U',
-    'postgres',
-    '-d',
-    'postgres',
-    '-q',
-    '-c',
-    'GRANT ALL ON public.page, public.page_section TO service_role; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO service_role; GRANT SELECT ON public.page, public.page_section TO anon, authenticated;',
-  ]);
+  run(
+    'docker',
+    [
+      'exec',
+      DB_CONTAINER,
+      'psql',
+      '-U',
+      'postgres',
+      '-d',
+      'postgres',
+      '-q',
+      '-c',
+      'GRANT ALL ON public.page, public.page_section TO service_role; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO service_role; GRANT SELECT ON public.page, public.page_section TO anon, authenticated;',
+    ],
+    { quiet: true }
+  );
   console.log(
-    `docs stack up (project ${PROJECT_ID}); next: pnpm local docs seed`
+    `docs stack up on ${stackEnv().API_URL}; next: pnpm local docs seed`
   );
 }
 
@@ -284,7 +320,10 @@ export async function main(argv: string[]) {
       cmdApi(values);
       break;
     case 'down':
-      run('supabase', ['stop', '--workdir', OVERLAY]);
+      run('supabase', ['stop', '--workdir', OVERLAY], { quiet: true });
+      console.log(
+        `docs stack stopped (project ${PROJECT_ID}); the seeded index stays in its docker volume`
+      );
       break;
     default:
       fail(usage);