diff --git a/.github/workflows/workspace-selftest.yml b/.github/workflows/workspace-selftest.yml new file mode 100644 index 00000000..8549a2b6 --- /dev/null +++ b/.github/workflows/workspace-selftest.yml @@ -0,0 +1,31 @@ +name: workspace-selftest + +# Free self-tests of the eval-source workspace glue (workspace/README.md): +# shell + node stdlib only — no API keys, no model spend, no docker, no +# pnpm install. Submodules are fetched so the hooks/ab tests exercise the +# real repos instead of skipping. +on: + pull_request: + paths: + - "workspace/**" + - "mise.toml" + - ".github/workflows/workspace-selftest.yml" + +jobs: + selftest: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + submodules: recursive + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + - name: status self-test + run: bash workspace/scripts/status.test.sh + - name: hooks self-test + run: bash workspace/scripts/hooks.test.sh + - name: ab self-test + run: bash workspace/scripts/ab.test.sh diff --git a/.gitignore b/.gitignore index 932a6aea..2154efe1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,8 @@ dist/ results/*/ .sync-tmp/ +# eval-source workspace glue (workspace/README.md) +/supabase/ +/results-ab/ +/.docs-index-stamp.json +/.publish/ diff --git a/.gitmodules b/.gitmodules index 094026de..f3863093 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,4 @@ [submodule "submodules/mcp"] path = submodules/mcp url = git@github.com:supabase/mcp.git + ignore = all diff --git a/README.md b/README.md index ebf6991d..85761803 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,16 @@ Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/co - **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network) under `.claude/skills/`. When a task matches, the agent reads `.claude/skills//SKILL.md` (and any files it references) with its file tools. - **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name. +## Eval-source workspace (docs / skills / MCP loops) + +The glue for testing changes to the agent's *inputs* — docs pages, skills, +and MCP server source — lives in [`workspace/`](workspace/README.md): edit a +source, run the affected evals against the local change, or run a head-to-head +A/B (`mise run ab`) with per-arm provenance receipts. Tasks are driven by +`mise` from the repo root (`mise run status` is the bootstrap probe). The +docs monorepo stays an opt-in sparse clone; the MCP server is the pinned +`submodules/mcp` (see "Running against an exact MCP server revision" above). + ## Framework Checks ```bash diff --git a/demo/canary-eval/EVAL.ts b/demo/canary-eval/EVAL.ts new file mode 100644 index 00000000..baa58e40 --- /dev/null +++ b/demo/canary-eval/EVAL.ts @@ -0,0 +1,19 @@ +import { type CheckResult, type ToolScorer } from '@supabase-evals/core'; + +// eval-workspace docs-discriminator demo: '@supabase/pinniped' is a fictional package +// planted ONLY in the local docs by scripts/ab-demo.sh. It cannot come from model +// priors or CLI scaffolding, so naming it proves the agent retrieved the answer +// from the local docs index — and removing it (the A/B baseline) proves the doc +// was the cause. +const scorer: ToolScorer = async (ctx) => { + const report = ctx.agentReport ?? ''; + const checks: CheckResult[] = [ + { + name: 'named the docs-only package @supabase/pinniped', + passed: /pinniped/i.test(report), + }, + ]; + return { passed: checks[0].passed, checks }; +}; + +export default scorer; diff --git a/demo/canary-eval/PROMPT.md b/demo/canary-eval/PROMPT.md new file mode 100644 index 00000000..b3b97423 --- /dev/null +++ b/demo/canary-eval/PROMPT.md @@ -0,0 +1,13 @@ +--- +stage: investigate +suite: other +interface: mcp +product: + - edge-functions +topic: + - sdk +--- + +I'm building a Supabase-managed cron/queue worker on the internal "Nimbus" +runtime. Search the Supabase documentation and tell me the exact JavaScript +helper package it recommends for this case. Name the package precisely. diff --git a/mise.toml b/mise.toml new file mode 100644 index 00000000..bf1dc244 --- /dev/null +++ b/mise.toml @@ -0,0 +1,107 @@ +# supabase/evals — mise tasks (Supabase uses mise throughout; never a justfile). +# The eval-source workspace glue lives in workspace/ (see workspace/README.md): +# it wires docs, skills, and MCP-server sources into this harness for +# edit -> eval loops. Tasks run from the repo root. Flag-style args need `--`: +# mise run eval -- --eval --experiment + +[tools] +node = "22" +pnpm = "10.24.0" + +[tasks.setup] +description = "Install deps, init submodules (agent-skills + mcp), apply patches, wire .env, print status" +run = "workspace/scripts/setup.sh" + +[tasks.status] +description = "Host repo + submodule + clone state, env keys present, tooling" +run = "workspace/scripts/status.sh" + +[tasks.eval] +description = "Run evals with the workspace env applied (args pass to `pnpm eval`)" +run = "workspace/scripts/eval.sh" + +[tasks.update] +description = "Fetch + fast-forward the supabase clone, re-applying patches (-- --check = report only)" +run = "workspace/scripts/update.sh" + +[tasks.publish] +description = "Clean PR branch from patched-repo work: publish [--with ] (or --list)" +run = "workspace/scripts/publish.sh" + +[tasks.hooks-test] +description = "Self-test of the pre-push guard lifecycle (install/chain/reinstall; fixtures only, self-cleaning)" +run = "workspace/scripts/hooks.test.sh" + +[tasks.affected] +description = "Map changed skill/docs/mcp paths to a ready-to-run eval command" +run = "workspace/scripts/affected-task.sh" + +[tasks.ab] +description = "Head-to-head: [experiment] — edit applied vs reverted. No args = readiness probe" +run = "workspace/scripts/ab-task.sh" + +[tasks.ab-demo] +description = "Guided LIVE demo of the docs A/B loop (plants a canary doc, runs the real A/B, cleans up; asks before spending)" +run = "workspace/scripts/ab-demo.sh" + +[tasks.ab-test] +description = "Zero-cost self-test of the A/B runner (fakes the eval; only touches a clean skill file)" +run = "workspace/scripts/ab.test.sh" + +[tasks.status-test] +description = "Self-test of status.sh's Next/Ready diagnosis against synthetic workspace states" +run = "workspace/scripts/status.test.sh" + +[tasks.store-key] +description = "Store an API key in the keychain: store-key " +run = "workspace/scripts/store-key.sh" + +[tasks.apply-patches] +description = "(Re)apply the tracked enabler patches into the patched repos; idempotent" +run = "workspace/scripts/apply-patches.sh" + +# --- MCP-server loop (optional; not part of setup) --- + +[tasks.mcp-build] +description = "Build the mcp submodule with the enabler patches applied" +# Order matters: `git submodule update` on an initialized submodule re-checks +# out the pin, ORPHANING the patch marker commit (and any user commits) — so +# init strictly BEFORE apply-patches, and never again after (which is why this +# does not delegate to the root `pnpm mcp:build`, whose first step is another +# submodule update; that script is the evals-native unpatched flow). +run = "git submodule update --init submodules/mcp && pnpm --dir submodules/mcp install && workspace/scripts/apply-patches.sh && pnpm --dir submodules/mcp build" + +[tasks.mcp-eval] +description = "Run evals against the local mcp build (auto init+patch+build)" +depends = ["mcp-build"] +run = "workspace/scripts/mcp-eval.sh" + +# --- Docs loop (optional, heavy — needs Docker + the supabase CLI) --- + +[tasks.clone-docs] +description = "Sparse-clone supabase/supabase (apps/docs + deps) and install. Run once" +run = "workspace/scripts/clone-docs.sh" + +[tasks.docs-up] +description = "Start the local content DB (ports 55321+) with docs migrations" +depends = ["clone-docs"] +run = "workspace/scripts/apply-patches.sh && workspace/scripts/docs-up.sh" + +[tasks.docs-down] +description = "Stop the local content DB" +run = "workspace/scripts/docs-down.sh" + +[tasks.docs-api] +description = "Serve the docs content GraphQL API at :3001 for search_docs" +depends = ["docs-up"] +run = "workspace/scripts/docs-api.sh" + +[tasks.docs-seed] +description = "Full docs embed — spends OpenAI credits; asks to confirm. Run once" +depends = ["docs-up"] +run = "workspace/scripts/docs-seed.sh" + +[tasks.docs-index] +description = "Incremental re-embed of changed docs pages (checksum-based, fail-closed)" +depends = ["docs-up"] +run = "workspace/scripts/docs-index.sh" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e25fc37b..8ef5af86 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -906,6 +906,7 @@ export function supabaseMcpServer( options: { features?: string[]; version?: string; + contentApiUrl?: string; } = {} ): McpServerDefinition { const features = options.features ?? [ @@ -938,6 +939,17 @@ export function supabaseMcpServer( if (apiUrl) serverArgs.push('--api-url', apiUrl); const local = resolveLocalMcpServer(); + // Alternative docs Content API endpoint (e.g. a locally built docs + // index). Only a server that understands --content-api-url can accept + // it (the SUPABASE_MCP_SERVER_PATH build; the published 0.8.1 npx + // package rejects unknown flags), so the ENV fallback applies only + // alongside the local override — a stray env var can't break a plain + // npx run. The explicit option is intentional and always honored. + const contentApiUrl = + options.contentApiUrl ?? + (local ? process.env.SUPABASE_CONTENT_API_URL : undefined); + if (contentApiUrl) serverArgs.push('--content-api-url', contentApiUrl); + if (local) { // `node`, not process.execPath: CLI agents run this command INSIDE the // sandbox container, where the host's node binary path does not exist. diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts index bd3f81c3..1c69e023 100644 --- a/packages/core/src/mcp-server.test.ts +++ b/packages/core/src/mcp-server.test.ts @@ -27,6 +27,7 @@ import { // Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test. function clearEnv() { vi.stubEnv('SUPABASE_MCP_SERVER_PATH', undefined); + vi.stubEnv('SUPABASE_CONTENT_API_URL', undefined); } // A real on-disk build layout: the override path is existence-checked, so the @@ -57,6 +58,35 @@ describe('supabaseMcpServer().createConfig', () => { `@supabase/mcp-server-supabase@${MCP_SERVER_VERSION}` ); expect(config.args).toContain('--api-url'); + expect(config.args).not.toContain('--content-api-url'); + }); + + it('threads --content-api-url from the env var on the local override path', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir); + vi.stubEnv('SUPABASE_CONTENT_API_URL', 'https://env.test/gql'); + const { config } = await supabaseMcpServer().createConfig({}); + const i = config.args.indexOf('--content-api-url'); + expect(i).toBeGreaterThan(-1); + expect(config.args[i + 1]).toBe('https://env.test/gql'); + }); + + it('ignores a stray env var on the npx path (0.8.1 rejects unknown flags)', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_CONTENT_API_URL', 'https://env.test/gql'); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.command).toBe('npx'); + expect(config.args).not.toContain('--content-api-url'); + }); + + it('prefers the explicit contentApiUrl option over the env var', async () => { + clearEnv(); + vi.stubEnv('SUPABASE_CONTENT_API_URL', 'https://env.test/gql'); + const { config } = await supabaseMcpServer({ + contentApiUrl: 'https://opt.test/gql', + }).createConfig({}); + const i = config.args.indexOf('--content-api-url'); + expect(config.args[i + 1]).toBe('https://opt.test/gql'); }); it('launches a local build dir with node when SUPABASE_MCP_SERVER_PATH is set', async () => { diff --git a/workspace/README.md b/workspace/README.md new file mode 100644 index 00000000..bae2074f --- /dev/null +++ b/workspace/README.md @@ -0,0 +1,87 @@ +# Eval-source workspace + +Glue that wires the agent's **inputs** — docs, skills, and the MCP server — +into this harness, so you can change an input and measure the effect on evals: +edit a skill, a docs page, or MCP server source, then run the affected evals +against the local change. Formerly the standalone +[eval-workspace](https://github.com/supabase/eval-workspace) repo; folded in +here so the sources under test live beside the harness that tests them +(direction agreed in the workspace-layout Slack thread, 2026-07-23). + +All tasks run via `mise` from the repo root (Supabase convention; `mise.toml` +lives there). Flag-style args need `--`: `mise run eval -- --eval `. + +## Bootstrap + +```bash +mise run status # the state probe: prints the exact next command for anything missing +mise run setup # idempotent: install, init submodules (agent-skills + mcp), patches, .env +``` + +Keys live in the macOS keychain as `eval-workspace:` — add them with +`mise run store-key ` (hidden +prompt, immune to the 128-char truncation of raw `security -w`). Non-macOS: +put keys in the repo-root `.env` (the fallback `status` will route you to). + +`mise run ab-test` is the zero-cost self-check that the glue works. + +## The three loops + +| Loop | Source | Sync after an edit | +|---|---|---| +| Skills | `submodules/agent-skills` (a working tree: edit in place) | none | +| MCP server | `submodules/mcp` | `mise run mcp-build` | +| Docs | `supabase/` (opt-in sparse clone: `mise run clone-docs`) | `mise run docs-index` (cents) | + +- **MCP loop**: `mise run mcp-eval -- ` builds the submodule (with the + enabler patches) and runs evals against it via `SUPABASE_MCP_SERVER_PATH`. +- **Docs loop** (heavy: Docker + supabase CLI): `docs-up` → `docs-seed` (spends + OpenAI money once, ~$0.12; always confirm with the user first) → `docs-api` + serves the content GraphQL API for `search_docs`; `docs-index` re-embeds + changed pages incrementally. To measure a docs edit, point an eval run at + the local index through the local server build: + `SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql mise run mcp-eval -- --eval ` + (or just use `mise run ab`, which wires this automatically). Content DB + ports are 55321+ to avoid the eval local-stack range (54321-9). Measuring + docs impact needs a tools-mode (`interface: mcp`) eval whose answer lives + only in the docs. + +## Head-to-head A/B + +Measure whether ONE edit moves an eval: make a tracked, unstaged edit in a +loop's scope, then + +```bash +mise run ab # treatment (edit applied) vs baseline (edit reverted) +``` + +The edit is always restored (a failed restore fails the run loudly); per-arm +provenance receipts land in `results-ab/*.json`. Cost: two model runs. No +args = readiness probe. First time? `mise run ab-demo` is a guided, +self-cleaning live proof on the docs loop (spend-gated, asks first). + +## Patches & publishing + +Local changes to the patched repos (the `supabase/` clone and the +`submodules/mcp` working tree) are tracked as `.patch` files in +`workspace/patches/` and applied as marker commits — see +[patches/README.md](./patches/README.md) for the manifest and the +publish flow (`mise run publish `). A pre-push guard in each +patched repo blocks marker commits from leaving the machine; the host repo +never gets hooks or marker commits. + +## Provenance + +`mise run status -- --json` prints a receipt: host repo SHA + dirty state, +submodule pins, supabase clone state, patch fingerprints, and the docs-index +stamp (`.docs-index-stamp.json`, scoped to repo docs content only). `ab.sh` +embeds a per-arm copy into every A/B result, so a wrong-baseline run is +immediately obvious. + +## Self-tests (all free: no keys, no model spend) + +```bash +mise run status-test # status.sh Next/Ready diagnosis against synthetic states +mise run hooks-test # pre-push guard lifecycle +mise run ab-test # A/B runner with a faked eval +``` diff --git a/workspace/manifest.json b/workspace/manifest.json new file mode 100644 index 00000000..38858b1d --- /dev/null +++ b/workspace/manifest.json @@ -0,0 +1,28 @@ +{ + "repos": { + "mcp": { + "dir": "submodules/mcp", + "kind": "submodule", + "patches": ["mcp-content-api-url"] + }, + "supabase": { + "dir": "supabase", + "remote": "git@github.com:supabase/supabase.git", + "patches": [ + "supabase-content-local-ports", + "supabase-docs-index-fail-closed", + "supabase-docs-guide-checksum", + "supabase-docs-lint-warnings-skip", + "supabase-docs-reference-dup-sources" + ], + "localPatches": [ + "supabase-content-local-ports", + "supabase-docs-lint-warnings-skip" + ] + }, + "skills": { + "dir": "submodules/agent-skills", + "kind": "submodule" + } + } +} diff --git a/workspace/patches/README.md b/workspace/patches/README.md new file mode 100644 index 00000000..0b54a8c5 --- /dev/null +++ b/workspace/patches/README.md @@ -0,0 +1,58 @@ +# Enabler patches + +Local changes to the patched repos (the `supabase/` clone and the +`submodules/mcp` working tree), materialized by +`workspace/scripts/apply-patches.sh` as identifiable **local commits** at the +bottom of each repo's branch: + + [eval-workspace-local] dev shim — must never leave this machine + [eval-workspace-upstream] upstream candidate — leaves ONLY via + `mise run publish --with ` + +Your own work sits **above** these as normal commits, so `git commit -am` can +never sweep plumbing into it, and `mise run ab` can A/B any file (plumbing is +not in the working diff). A pre-push guard in each patched repo blocks marker +commits from being pushed; a pre-existing pre-push hook is chained after the +guard. The host repo (this one) never gets hooks or marker commits. Deliberate +override (skips only the marker checks): `EVAL_WORKSPACE_ALLOW_PUSH=1 git push …`. + +The `.patch` files here are **canonical**: `apply-patches` builds each commit +from the patch via the index, verifies the staged diff equals the patch before +committing (a user edit in a patch-owned file can never be absorbed), and on +every later run verifies the existing marker commit still matches the file. +`workspace/manifest.json` is the single source of truth mapping patches → +repos / kinds (read via `workspace/scripts/manifest.mjs`; marker subjects +derive from the kind). + +| Patch | Repo | Files | Kind | Tests | What | +|---|---|---|---|---|---| +| `mcp-content-api-url` | `submodules/mcp` | `transports/stdio.ts` | upstream | 3 stdio integration tests (upstream) | `--content-api-url` flag + `SUPABASE_CONTENT_API_URL` — **merged upstream ([mcp#343](https://github.com/supabase/mcp/pull/343), 2026-07-23), not yet in a release**; the patch retires when a release ships the flag and the pin moves past it | +| `supabase-content-local-ports` | supabase/supabase | `config.toml` | **LOCAL-ONLY** | n/a | dev ports 55321+ (avoid evals' 54321-9) | +| `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report | +| `supabase-docs-guide-checksum` | supabase/supabase | `guideModelLoader.ts` | upstream | none | guide checksum + `tryCatch` `onError` fix (docs-index edit detection) | +| `supabase-docs-lint-warnings-skip` | supabase/supabase | `lint-warnings-guide.ts` | **LOCAL-ONLY** | n/a | `DOCS_EMBED_ALLOW_MISSING_SOURCES` skip | +| `supabase-docs-reference-dup-sources` | supabase/supabase | `sources/reference-doc.ts` + test | upstream | **unit + real-output tests** | dedupe duplicate reference source paths (the 18-page `inserted 2/1` fix) | + +## Changing or regenerating a patch + +The marker commit and the `.patch` file must stay in sync — `apply-patches` +verifies both directions and fails loudly on drift. + +- **You changed the plumbing in the clone** (amended/edited the marker commit): + refresh the canonical file from the commit — + `git -C diff ^ > workspace/patches/.patch` + (plain `git diff` cannot see committed changes). +- **You edited the `.patch` file**: drop the old marker commit + (`git -C rebase --onto ^ `), then + `mise run apply-patches` re-creates it from the file. +- **A patch adds brand-new files**: when generating, `git add -N ` first, + then `git reset -- ` after — a leftover intent-to-add entry breaks `git stash`. + +## Upstreaming order + +`mcp-content-api-url` — merged upstream ([mcp#343](https://github.com/supabase/mcp/pull/343)); retires at the next mcp release + pin bump. +Then the three upstream supabase fixes (independent; `guide-checksum` and +`index-fail-closed` still need regression tests; `reference-dup-sources` is +test-covered and PR-ready). The two former evals patches are done: the +local-build override shipped with evals#109, and the contentApiUrl threading +landed as a normal commit on this branch. diff --git a/workspace/patches/mcp-content-api-url.patch b/workspace/patches/mcp-content-api-url.patch new file mode 100644 index 00000000..f29c6bc7 --- /dev/null +++ b/workspace/patches/mcp-content-api-url.patch @@ -0,0 +1,40 @@ +diff --git a/packages/mcp-server-supabase/src/transports/stdio.ts b/packages/mcp-server-supabase/src/transports/stdio.ts +index e974296..991c46f 100644 +--- a/packages/mcp-server-supabase/src/transports/stdio.ts ++++ b/packages/mcp-server-supabase/src/transports/stdio.ts +@@ -16,6 +16,7 @@ async function main() { + ['project-ref']: projectId, + ['read-only']: readOnly, + ['api-url']: apiUrl, ++ ['content-api-url']: cliContentApiUrl, + ['version']: showVersion, + ['features']: cliFeatures, + }, +@@ -34,6 +35,9 @@ async function main() { + ['api-url']: { + type: 'string', + }, ++ ['content-api-url']: { ++ type: 'string', ++ }, + ['version']: { + type: 'boolean', + }, +@@ -59,6 +63,9 @@ async function main() { + + const features = cliFeatures ? parseList(cliFeatures) : undefined; + ++ const contentApiUrl = ++ cliContentApiUrl ?? process.env.SUPABASE_CONTENT_API_URL; ++ + const platform = createSupabaseApiPlatform({ + accessToken, + apiUrl, +@@ -69,6 +76,7 @@ async function main() { + projectId, + readOnly, + features, ++ contentApiUrl, + }); + + const transport = new StdioServerTransport(); diff --git a/workspace/patches/supabase-content-local-ports.patch b/workspace/patches/supabase-content-local-ports.patch new file mode 100644 index 00000000..d6ee5bc4 --- /dev/null +++ b/workspace/patches/supabase-content-local-ports.patch @@ -0,0 +1,46 @@ +diff --git a/supabase/config.toml b/supabase/config.toml +index b03bf67c0d..e8f75e262b 100644 +--- a/supabase/config.toml ++++ b/supabase/config.toml +@@ -1,13 +1,13 @@ + # A string used to distinguish different Supabase projects on the same host. Defaults to the working + # directory name when running `supabase init`. +-project_id = "supabase" ++project_id = "eval-workspace-content" + + [remotes.prod] + project_id = "xguihxuzqibwxjnimxev" + + [api] + # Port to use for the API URL. +-port = 54321 ++port = 55321 + # Schemas to expose in your API. Tables, views and functions in this schema will get API + # endpoints. public and storage are always included. + schemas = ["public", "content", "storage", "graphql_public"] +@@ -22,14 +22,14 @@ schemas = ["public", "content", "storage", "graphql_public"] + + [db] + # Port to use for the local database URL. +-port = 54322 ++port = 55322 + # The database major version to use. This has to be the same as your remote database's. Run `SHOW + # server_version;` on the remote database to check. + major_version = 15 + + [studio] + # Port to use for Supabase Studio. +-port = 54323 ++port = 55323 + + openai_api_key = "env(OPENAI_API_KEY)" + +@@ -37,7 +37,7 @@ openai_api_key = "env(OPENAI_API_KEY)" + # are monitored, and you can view the emails that would have been sent from the web interface. + [inbucket] + # Port to use for the email testing server web interface. +-port = 54324 ++port = 55324 + + [storage] + # The maximum file size allowed (e.g. "5MB", "500KB"). diff --git a/workspace/patches/supabase-docs-guide-checksum.patch b/workspace/patches/supabase-docs-guide-checksum.patch new file mode 100644 index 00000000..c85530f6 --- /dev/null +++ b/workspace/patches/supabase-docs-guide-checksum.patch @@ -0,0 +1,44 @@ +diff --git a/apps/docs/resources/guide/guideModelLoader.ts b/apps/docs/resources/guide/guideModelLoader.ts +index 551bd98abf..219b2450a1 100644 +--- a/apps/docs/resources/guide/guideModelLoader.ts ++++ b/apps/docs/resources/guide/guideModelLoader.ts +@@ -1,6 +1,7 @@ + import matter from 'gray-matter' + import { promises as fs } from 'node:fs' + import { join, relative, resolve } from 'node:path' ++import { createHash } from 'node:crypto' + + import { extractMessageFromAnyError, FileNotFoundError, MultiError } from '~/app/api/utils' + import { preprocessMdxWithDefaults } from '~/features/directives/utils' +@@ -117,6 +118,11 @@ export class GuideModelLoader { + // Process MDX to get chunked sections for embedding + const { sections } = await processMdx(processedContent) + ++ // Content checksum so docs-index can detect guide edits (without this, ++ // GuideModel had no checksum -> normalized to '' -> every guide compared ++ // equal forever and incremental re-embed never fired for guides). ++ const checksum = createHash('sha256').update(processedContent).digest('base64') ++ + // Create subsections from the chunked sections + const subsections = sections.map((section) => ({ + title: section.heading, +@@ -133,6 +139,7 @@ export class GuideModelLoader { + return new GuideModel({ + title, + href, ++ checksum, + content: processedContent, + metadata, + subsections, +@@ -140,9 +147,9 @@ export class GuideModelLoader { + }, + (error) => { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { +- throw new FileNotFoundError('', error) ++ return new FileNotFoundError('', error) + } +- throw new Error( ++ return new Error( + `Failed to load guide from ${relPath}: ${extractMessageFromAnyError(error)}`, + { + cause: error, diff --git a/workspace/patches/supabase-docs-index-fail-closed.patch b/workspace/patches/supabase-docs-index-fail-closed.patch new file mode 100644 index 00000000..a1322c0e --- /dev/null +++ b/workspace/patches/supabase-docs-index-fail-closed.patch @@ -0,0 +1,96 @@ +diff --git a/apps/docs/scripts/search/generate-embeddings.ts b/apps/docs/scripts/search/generate-embeddings.ts +index d7519d2b15..4dfe6e6ba6 100644 +--- a/apps/docs/scripts/search/generate-embeddings.ts ++++ b/apps/docs/scripts/search/generate-embeddings.ts +@@ -118,6 +118,7 @@ function initSupabase(): SupabaseClient { + type PreparedSections = { + allSectionsToProcess: PageSectionForEmbedding[] + pageInfoMap: Map ++ preparationFailureCount: number + } + + async function prepareSections( +@@ -135,6 +136,7 @@ async function prepareSections( + + const allSectionsToProcess: PageSectionForEmbedding[] = [] + const pageInfoMap = new Map() ++ let preparationFailureCount = 0 + + for (const sourceBatch of createBatches(embeddingSources, CONFIG.SOURCE_CONCURRENCY)) { + await Promise.all( +@@ -238,6 +240,7 @@ async function prepareSections( + } catch (err) { + console.error(`Error preparing path '${path}' for processing.`) + console.error(err) ++ preparationFailureCount++ + } + }) + ) +@@ -246,7 +249,7 @@ async function prepareSections( + console.log( + `Prepared ${allSectionsToProcess.length} sections for processing from ${pageInfoMap.size} pages` + ) +- return { allSectionsToProcess, pageInfoMap } ++ return { allSectionsToProcess, pageInfoMap, preparationFailureCount } + } + + async function processAndInsertEmbeddings( +@@ -321,6 +324,9 @@ type BatchEmbeddingResult = { + processedCount: number + } + ++// Running total of embedding tokens billed across all batches, for a cost report. ++let embeddingTokenTotal = 0 ++ + async function processEmbeddingBatch( + openai: OpenAI, + batch: PageSectionForEmbedding[], +@@ -376,6 +382,7 @@ async function processEmbeddingBatch( + // Replace inputs with truncated inputs for downstream bookkeeping + for (let i = 0; i < inputs.length; i++) inputs[i] = truncatedInputs[i] + } ++ embeddingTokenTotal += embeddingResponse.usage?.total_tokens ?? 0 + + const { sectionsWithEmbeddings, failedSectionIndexes } = mapEmbeddingsToSections( + batch, +@@ -467,7 +474,7 @@ async function generateEmbeddings() { + : 'Checking which pages are new or have changed' + ) + +- const { allSectionsToProcess, pageInfoMap } = await prepareSections( ++ const { allSectionsToProcess, pageInfoMap, preparationFailureCount } = await prepareSections( + supabaseClient, + pageTable, + pageSectionTable, +@@ -493,10 +500,13 @@ async function generateEmbeddings() { + console.log( + `Page summary: ${processingResult.successfulPages.size} successful, ${processingResult.failedPages.size} failed` + ) ++ console.log( ++ `Embedding tokens billed: ${embeddingTokenTotal} (~$${((embeddingTokenTotal / 1_000_000) * 0.1).toFixed(4)} at $0.10/M for ${CONFIG.EMBEDDING_MODEL})` ++ ) + } catch (error) { + console.error('Critical error during embedding processing:', error) + console.log('Exiting due to complete processing failure') +- return ++ throw error + } + + console.log(`\nUpdating checksums for ${processingResult.successfulPages.size} successful pages`) +@@ -512,6 +522,16 @@ async function generateEmbeddings() { + + logFailedPages(pageInfoMap, processingResult) + ++ if ( ++ preparationFailureCount > 0 || ++ processingResult.failedPages.size > 0 || ++ successfulChecksumUpdates !== processingResult.successfulPages.size ++ ) { ++ throw new Error( ++ `Index incomplete: ${preparationFailureCount} source preparation failure(s), ${processingResult.failedPages.size} page failure(s), and ${processingResult.successfulPages.size - successfulChecksumUpdates} checksum update failure(s); refusing to purge old pages` ++ ) ++ } ++ + await purgeOldPages(supabaseClient, pageTable, refreshVersion) + + console.log('Embedding generation complete') diff --git a/workspace/patches/supabase-docs-lint-warnings-skip.patch b/workspace/patches/supabase-docs-lint-warnings-skip.patch new file mode 100644 index 00000000..70a3bf71 --- /dev/null +++ b/workspace/patches/supabase-docs-lint-warnings-skip.patch @@ -0,0 +1,15 @@ +diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts +index d28a57a27f..0207f87504 100644 +--- a/apps/docs/scripts/search/sources/lint-warnings-guide.ts ++++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts +@@ -29,6 +29,10 @@ export class LintWarningsGuideLoader extends BaseLoader { + + async load() { + if (!appId || !installationId || !privateKey) { ++ if (process.env.DOCS_EMBED_ALLOW_MISSING_SOURCES) { ++ console.warn('Skipping lint-warnings guide source: DOCS_GITHUB_APP_* not set (DOCS_EMBED_ALLOW_MISSING_SOURCES)') ++ return [] ++ } + throw new Error('Missing DOCS_GITHUB_APP_* environment variables') + } + diff --git a/workspace/patches/supabase-docs-reference-dup-sources.patch b/workspace/patches/supabase-docs-reference-dup-sources.patch new file mode 100644 index 00000000..b17ca5b3 --- /dev/null +++ b/workspace/patches/supabase-docs-reference-dup-sources.patch @@ -0,0 +1,135 @@ +diff --git a/apps/docs/scripts/search/sources/reference-doc.test.ts b/apps/docs/scripts/search/sources/reference-doc.test.ts +new file mode 100644 +index 0000000000..38f5cc9c0b +--- /dev/null ++++ b/apps/docs/scripts/search/sources/reference-doc.test.ts +@@ -0,0 +1,100 @@ ++import { mkdtemp, rm, writeFile } from 'node:fs/promises' ++import { existsSync } from 'node:fs' ++import { tmpdir } from 'node:os' ++import { join } from 'node:path' ++import { afterAll, describe, expect, it } from 'vitest' ++ ++import { loadClientLibReferenceFromNewPipeline } from './reference-doc' ++ ++/** ++ * Regression tests for duplicate entries in the new reference pipeline's ++ * sections.json: some trees repeat an entry (a category node and its overview ++ * child share id + slug), which used to emit two sources for one page path. ++ * Both sections then inserted under one page while the page's expected count ++ * came from the last source only, so the embed script flagged the page failed ++ * on every run (inserted 2/1) and the index could never complete. ++ */ ++ ++const tmpDirs: string[] = [] ++afterAll(async () => { ++ await Promise.all(tmpDirs.map((dir) => rm(dir, { recursive: true, force: true }))) ++}) ++ ++async function fixture(sections: unknown, functions: unknown): Promise { ++ const dir = await mkdtemp(join(tmpdir(), 'reference-doc-test-')) ++ tmpDirs.push(dir) ++ await writeFile(join(dir, 'sections.json'), JSON.stringify(sections)) ++ await writeFile(join(dir, 'functions.json'), JSON.stringify(functions)) ++ await writeFile(join(dir, 'typeSpec.json'), JSON.stringify({ methods: {} })) ++ return dir ++} ++ ++describe('loadClientLibReferenceFromNewPipeline', () => { ++ it('emits one source when a category node and its child repeat id + slug', async () => { ++ const contentDir = await fixture( ++ [ ++ { ++ id: 'auth-admin', ++ slug: 'auth-admin', ++ title: 'Auth Admin', ++ type: 'function', ++ items: [{ id: 'auth-admin', slug: 'auth-admin', title: 'Overview', type: 'function' }], ++ }, ++ ], ++ [{ id: 'auth-admin', title: 'Auth Admin', description: 'admin methods' }] ++ ) ++ ++ const sources = await loadClientLibReferenceFromNewPipeline({ ++ source: 'js-lib', ++ path: '/reference/javascript', ++ meta: { title: 'JavaScript Reference' }, ++ contentDir, ++ }) ++ ++ expect(sources.map((s) => s.path)).toEqual(['/reference/javascript/auth-admin']) ++ }) ++ ++ it('keeps the first flattened section when different ids share a slug', async () => { ++ const contentDir = await fixture( ++ [ ++ { id: 'first-id', slug: 'dup-slug', title: 'First', type: 'function' }, ++ { id: 'second-id', slug: 'dup-slug', title: 'Second', type: 'function' }, ++ ], ++ [ ++ { id: 'first-id', title: 'First fn', description: 'a' }, ++ { id: 'second-id', title: 'Second fn', description: 'b' }, ++ ] ++ ) ++ ++ const sources = await loadClientLibReferenceFromNewPipeline({ ++ source: 'js-lib', ++ path: '/reference/javascript', ++ meta: { title: 'JavaScript Reference' }, ++ contentDir, ++ }) ++ ++ expect(sources).toHaveLength(1) ++ expect(sources[0].path).toBe('/reference/javascript/dup-slug') ++ // deterministic winner: the first flattened section (matches the renderer's ++ // getFlattenedSections(...).find(...) lookup, which also takes the first) ++ expect(sources[0].extractTitle()).toBe('First fn') ++ }) ++ ++ // The real generated outputs are what production embeds; assert the invariant ++ // the page upsert depends on (path uniqueness) holds for them. ++ it.each(['javascript', 'dart'])('emits unique page paths for %s/v2', async (lib) => { ++ const contentDir = `content/reference/${lib}/v2` ++ if (!existsSync(join(contentDir, 'sections.json'))) return // sparse checkout without generated content ++ ++ const sources = await loadClientLibReferenceFromNewPipeline({ ++ source: `${lib}-lib`, ++ path: `/reference/${lib}`, ++ meta: { title: `${lib} Reference` }, ++ contentDir, ++ }) ++ ++ const paths = sources.map((s) => s.path) ++ expect(new Set(paths).size).toBe(paths.length) ++ expect(paths.length).toBeGreaterThan(0) ++ }) ++}) +diff --git a/apps/docs/scripts/search/sources/reference-doc.ts b/apps/docs/scripts/search/sources/reference-doc.ts +index 2e433746fe..eb81c75bcf 100644 +--- a/apps/docs/scripts/search/sources/reference-doc.ts ++++ b/apps/docs/scripts/search/sources/reference-doc.ts +@@ -332,14 +332,23 @@ export async function loadClientLibReferenceFromNewPipeline({ + + const flattened = flattenSections(refSections) + ++ // sections.json repeats some entries (a category node and its overview child ++ // share id + slug), and both resolve to the same spec entry, so keeping the ++ // duplicate would create two sources for one page path: sections double-insert ++ // and the page's inserted/expected bookkeeping fails. Keep the first. ++ const seenPaths = new Set() ++ + return flattened + .map((refSection) => { + const specSection = enriched.find((e) => e.id === refSection.id) + if (!specSection) return undefined ++ const pagePath = `${path}/${refSection.slug}` ++ if (seenPaths.has(pagePath)) return undefined ++ seenPaths.add(pagePath) + const titleForMeta = specSection.title || refSection.title + return new ClientLibReferenceSource( + source, +- `${path}/${refSection.slug}`, ++ pagePath, + refSection, + specSection, + { ...meta, slug: specSection.id, methodName: titleForMeta } diff --git a/workspace/scripts/ab-demo.sh b/workspace/scripts/ab-demo.sh new file mode 100755 index 00000000..df73f3ee --- /dev/null +++ b/workspace/scripts/ab-demo.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Guided, self-cleaning LIVE demonstration of the docs A/B loop — the fastest +# way to see (and trust) the whole mechanism: +# +# 1. plants an un-guessable fact in ONE local docs page: a fictional package +# `@supabase/pinniped` for the fictional "Nimbus" runtime (append-only edit) +# 2. installs a matching throwaway eval (demo/canary-eval/) that PASSES only +# if the agent names that package +# 3. runs the real head-to-head (workspace/scripts/ab.sh): treatment (fact +# embedded in the local index) vs baseline (fact removed + re-embedded) +# 4. expected result: baseline FAIL 0/1 -> treatment PASS 1/1 — proof that a +# local docs edit alone moves an eval through search_docs +# 5. cleans up completely: guide reverted, canary de-embedded, eval removed +# +# Cost: 2 model runs (claude-sonnet-5) + a few embedding cents. Asks first. +set -euo pipefail +cd "$(dirname "$0")/../.." + +GUIDE=supabase/apps/docs/content/guides/auth/choosing-a-server-package.mdx +EVAL_ID=investigate-workspace-canary-nimbus-package +EVAL_DST=evals/$EVAL_ID + +fail() { echo "not ready: $1" >&2; echo "run \`mise run ab\` (no args) for the readiness probe + fix commands" >&2; exit 1; } + +# --- keys: direct keychain reads (load-keys can be flaky in foreground shells) --- +for k in ANTHROPIC_API_KEY OPENAI_API_KEY; do + v="$(security find-generic-password -a "$USER" -s "eval-workspace:$k" -w 2>/dev/null || true)" + [ -n "$v" ] && export "$k=$v" +done + +# --- hard preflight (same gates the A/B itself needs) --- +[ -e supabase/.git ] || fail "supabase not cloned" +[ -n "${ANTHROPIC_API_KEY:-}" ] || fail "ANTHROPIC_API_KEY missing" +[ -n "${OPENAI_API_KEY:-}" ] || fail "OPENAI_API_KEY missing" +docker exec supabase_db_eval-workspace-content true 2>/dev/null || fail "content DB not running" +pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) +[ "${pages:-0}" -gt 0 ] 2>/dev/null || fail "docs index not seeded" +curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql || fail "docs-api not serving on :3001" +git -C supabase diff --quiet -- "${GUIDE#supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}" +[ ! -e "$EVAL_DST" ] || fail "$EVAL_DST already exists — remove it first" + +echo "This runs a live A/B: 2 claude-sonnet-5 runs + a few embedding cents, ~3-5 min." +read -r -p "Type 'demo' to proceed: " confirmation +[ "$confirmation" = demo ] || { echo "cancelled."; exit 0; } + +# --- plant the canary (append-only; reverted by the cleanup trap) --- +cleanup() { + echo "== demo cleanup: reverting canary, de-embedding, removing throwaway eval ==" + git -C supabase checkout -- "${GUIDE#supabase/}" 2>/dev/null || true + rm -rf "$EVAL_DST" + workspace/scripts/docs-index.sh >/dev/null 2>&1 \ + || echo "WARNING: cleanup re-embed failed — run \`mise run docs-index\` to de-embed the canary" >&2 +} +trap cleanup EXIT + +cat >> "$GUIDE" <<'EOF' + +## Nimbus workers + +- You're on a **Supabase-managed cron/queue worker** (the internal "Nimbus" runtime) → use **`@supabase/pinniped`**, the dedicated helper for Nimbus workers. +EOF +mkdir -p "$EVAL_DST" +cp demo/canary-eval/PROMPT.md demo/canary-eval/EVAL.ts "$EVAL_DST/" + +echo "== canary planted in $GUIDE; running the head-to-head ==" +workspace/scripts/ab.sh "$EVAL_ID" claude-sonnet-5 "$GUIDE" + +echo +echo "What you just saw: the ONLY difference between the two runs was that one" +echo "local docs page contained the planted fact. treatment PASS + baseline FAIL" +echo "means the agent found it via search_docs against YOUR local index — the" +echo "edit->embed->eval loop works end to end. Results: results-ab/$EVAL_ID.*.json" diff --git a/workspace/scripts/ab-ready.sh b/workspace/scripts/ab-ready.sh new file mode 100755 index 00000000..9b375f68 --- /dev/null +++ b/workspace/scripts/ab-ready.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# A/B readiness probe: which loops can you head-to-head right now, what's +# missing, and the exact command to fix each gap. `mise run ab` with no args +# lands here. Read-only; never mutates anything. +set -euo pipefail +cd "$(dirname "$0")/../.." + +NEXT="" +ok() { printf ' [ok] %s\n' "$1"; } +miss() { printf ' [--] %s\n' "$1"; NEXT="${NEXT} $2\n"; } +have_key() { security find-generic-password -a "$USER" -s "eval-workspace:$1" >/dev/null 2>&1 || { [ -f .env ] && grep -qE "^$1=.+" .env; }; } + +echo "A/B readiness — mise run ab [experiment=claude-sonnet-5]" + +echo +echo "skills loop (edit submodules/agent-skills/skills/…):" +if [ -e submodules/agent-skills/.git ]; then ok "agent-skills submodule initialized"; else miss "agent-skills submodule not initialized" "mise run setup"; fi +if have_key ANTHROPIC_API_KEY; then ok "ANTHROPIC_API_KEY present"; else miss "ANTHROPIC_API_KEY missing" "mise run store-key ANTHROPIC_API_KEY"; fi + +echo +echo "mcp loop (edit submodules/mcp/packages/…):" +if [ -e submodules/mcp/.git ] && [ -d submodules/mcp/packages/mcp-server-supabase/dist ]; then + ok "local mcp submodule initialized + built" +else + miss "local mcp submodule not initialized/built" "mise run mcp-build" +fi + +echo +echo "docs loop (edit supabase/apps/docs/content/… pages):" +if [ -e supabase/.git ]; then ok "supabase (apps/docs) cloned"; else miss "supabase not cloned" "mise run clone-docs"; fi +if docker exec supabase_db_eval-workspace-content true 2>/dev/null; then + ok "content DB running" + pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) + if [ "${pages:-0}" -gt 0 ] 2>/dev/null; then ok "index seeded ($pages pages)"; else miss "index empty" "mise run docs-seed # one-time full embed (OpenAI \$; asks to confirm)"; fi +else + miss "content DB not running" "mise run docs-up" +fi +if curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql; then ok "docs-api serving :3001"; else miss "docs-api not serving" "mise run docs-api # keep it running in a separate terminal"; fi +if have_key OPENAI_API_KEY; then ok "OPENAI_API_KEY present"; else miss "OPENAI_API_KEY missing" "mise run store-key OPENAI_API_KEY"; fi + +echo +if [ -n "$NEXT" ]; then + echo "Next (for the loop you want):" + printf '%b' "$NEXT" | awk '!seen[$0]++' +else + echo "All loops ready." +fi +echo +echo "Then: make ONE edit (tracked file, unstaged), pick an eval (ls evals/), run:" +echo " mise run ab " +echo "Wiring self-test (free): mise run ab-test · guided live demo: mise run ab-demo" diff --git a/workspace/scripts/ab-task.sh b/workspace/scripts/ab-task.sh new file mode 100755 index 00000000..7861cc41 --- /dev/null +++ b/workspace/scripts/ab-task.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# mise task wrapper: ab [experiment=claude-sonnet-5]. +# No args -> the readiness probe (which loops are runnable + how to fix gaps). +# Reorders to ab.sh's . For multiple edited +# files or other advanced use, call workspace/scripts/ab.sh directly. +set -euo pipefail +cd "$(dirname "$0")/../.." + +[ $# -gt 0 ] || exec workspace/scripts/ab-ready.sh +[ $# -ge 2 ] || { echo "usage: mise run ab [experiment] (no args = readiness probe)" >&2; exit 2; } +exec workspace/scripts/ab.sh "$1" "${3:-claude-sonnet-5}" "$2" diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh new file mode 100755 index 00000000..f4229f72 --- /dev/null +++ b/workspace/scripts/ab.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# Head-to-head eval: baseline (your edit reverted) vs treatment (edit applied). +# Same eval, same experiment; the ONLY difference is your uncommitted edit. +# +# Usage: workspace/scripts/ab.sh [more-paths...] +# is a file inside a clone; its clone selects the loop + how to +# re-sync between states: +# supabase/apps/docs/content/… docs loop (re-embed via docs-index; needs `mise run docs-api` up) +# submodules/mcp/… mcp loop (rebuild the local server) +# submodules/agent-skills/… skills loop (no re-sync; read live via symlink) +# +# The enabler patches live as [eval-workspace-*] commits below your work (see +# apply-patches.sh), so your unstaged edit is the ONLY working diff — stashing +# it for baseline can't disturb the plumbing, even on patch-owned files. +# +# Sync steps keep their normal exit semantics: if docs-index or the mcp build +# fails, the A/B aborts (and the restore trap puts your edit back). No special +# tolerance — a failing sync means the states can't be trusted. +# +# AB_DRYRUN=1 prints the resolved plan and exits (no runs, no side effects). +# AB_EVAL_CMD / AB_SYNC_CMD override the eval / sync step (testing; see ab.test.sh). +set -euo pipefail +cd "$(dirname "$0")/../.." + +[ $# -ge 3 ] || { echo "usage: workspace/scripts/ab.sh [more-paths...]" >&2; exit 2; } +EVAL="$1"; EXP="$2"; shift 2 +PATHS=("$@") + +# --- keys: direct keychain reads (robust vs load-keys foreground flakiness) --- +for k in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do + v="$(security find-generic-password -a "$USER" -s "eval-workspace:$k" -w 2>/dev/null || true)" + [ -n "$v" ] && export "$k=$v" +done + +# --- clone + loop from the first path --- +case "${PATHS[0]}" in + supabase/apps/docs/content/*) LOOP=docs; CLONE=supabase; STRIP=supabase/; PREFIX=supabase/apps/docs/content/ ;; + supabase/apps/docs/*) echo "docs A/B works on content pages (supabase/apps/docs/content/…) — other docs files aren't part of the embed loop: ${PATHS[0]}" >&2; exit 2 ;; + submodules/mcp/*) LOOP=mcp; CLONE=submodules/mcp; STRIP=submodules/mcp/; PREFIX=submodules/mcp/ ;; + submodules/agent-skills/*) LOOP=skills; CLONE=submodules/agent-skills; STRIP=submodules/agent-skills/; PREFIX=submodules/agent-skills/ ;; + *) echo "path must be under supabase/apps/docs/content/, submodules/mcp/, or submodules/agent-skills/: ${PATHS[0]}" >&2; exit 2 ;; +esac + +# every path must live in this loop's editable scope; build clone-relative paths +REL=() +for p in "${PATHS[@]}"; do + case "$p" in + "$PREFIX"*) REL+=("${p#$STRIP}") ;; + *) echo "all paths must be under $PREFIX (the loop chosen by ${PATHS[0]}): $p" >&2; exit 2 ;; + esac +done + +MCP="$PWD/submodules/mcp/packages/mcp-server-supabase" +CONTENT_URL="http://127.0.0.1:3001/docs/api/graphql" # also in affected.ts / docs-api.sh / workspace README — keep in sync +RUN_ENV=() +case "$LOOP" in + docs) RUN_ENV=( "SUPABASE_MCP_SERVER_PATH=$MCP" "SUPABASE_CONTENT_API_URL=$CONTENT_URL" ) ;; + mcp) RUN_ENV=( "SUPABASE_MCP_SERVER_PATH=$MCP" ) ;; +esac + +sync() { + if [ -n "${AB_SYNC_CMD:-}" ]; then bash -c "$AB_SYNC_CMD"; return; fi # override/test hook + case "$LOOP" in + docs) workspace/scripts/docs-index.sh ;; + mcp) ( cd submodules/mcp && pnpm build ) ;; + skills) : ;; + esac +} + +if [ -n "${AB_DRYRUN:-}" ]; then + echo "eval=$EVAL experiment=$EXP loop=$LOOP clone=$CLONE" + echo "revert paths: ${REL[*]}" + echo "run env: ${RUN_ENV[*]:-(none)}" + exit 0 +fi + +: "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY not in keychain — see README}" +# A/B reverts tracked, unstaged working-tree edits only, via a scoped git stash. +# `stash push -- ` merges whole-index state, so ANY staged entry in the +# clone (even an unrelated intent-to-add) breaks it. Require a clean index. +git -C "$CLONE" diff --cached --quiet --ita-visible-in-index \ + || { echo "the $CLONE index has staged changes (see: git -C $CLONE status) — unstage them first; ab's scoped stash needs a clean index" >&2; exit 1; } +for r in "${REL[@]}"; do + git -C "$CLONE" ls-files --error-unmatch -- "$r" >/dev/null 2>&1 \ + || { echo "not a tracked file in $CLONE: $r (ab reverts tracked edits; commit a brand-new file first, or revert it by hand)" >&2; exit 1; } + if git -C "$CLONE" diff --quiet -- "$r"; then + echo "no unstaged edit at $r (nothing to A/B)" >&2; exit 1 + fi +done + + +if [ "$LOOP" = docs ]; then + : "${OPENAI_API_KEY:?OPENAI_API_KEY not in keychain — the docs re-embed needs it}" + curl -sf -o /dev/null "$CONTENT_URL" || { echo "docs-api not reachable on :3001 — run \`mise run docs-api\` in another terminal first" >&2; exit 1; } + [ -d "$MCP/dist" ] || { echo "building local mcp (needed for search_docs routing)…"; ( cd submodules/mcp && pnpm install && pnpm build ); } +fi + +RES="results/$EXP/$EVAL.json" +OUT="results-ab"; mkdir -p "$OUT" + +run_eval() { # $1 = label + sync + if [ -n "${AB_EVAL_CMD:-}" ]; then + RES="$RES" AB_LABEL="$1" bash -c "$AB_EVAL_CMD" # override/test hook; must write $RES + else + env ${RUN_ENV[@]+"${RUN_ENV[@]}"} pnpm eval --eval "$EVAL" --experiment "$EXP" --runs 1 + fi + [ -f "$RES" ] || { echo "no result at $RES — check the eval/experiment ids" >&2; exit 1; } + cp "$RES" "$OUT/$EVAL.$1.json" + # per-arm receipt: capture provenance AFTER this arm's sync, so baseline and + # treatment records differ by exactly the edit under test (report-only). + # Same env as the run itself, or the receipt would claim mcp_override: null + # while the eval actually ran against the local build. + env ${RUN_ENV[@]+"${RUN_ENV[@]}"} node workspace/scripts/provenance.mjs --embed "$OUT/$EVAL.$1.json" +} + +# Restoration is ONE idempotent path: pop the stash AND re-sync, so the index/ +# build never stays at baseline while the tree shows treatment. The EXIT trap +# runs it on any post-stash failure, preserving the original exit status. +STASHED=0 +restore() { + [ "$STASHED" = 1 ] || return 0 + echo "== restoring edit ==" + if git -C "$CLONE" stash pop -q; then + STASHED=0 + else + echo "ERROR: stash pop failed — your edit is in: git -C $CLONE stash list" >&2 + return 1 + fi + if ! sync; then + echo "ERROR: post-restore re-sync failed — the index/build does not reflect your edit (docs: workspace/scripts/docs-index.sh; mcp: pnpm -C submodules/mcp build)" >&2 + return 1 + fi +} +# Preserve the run's own failure status; a clean run that fails cleanup exits nonzero. +trap 'rc=$?; if ! restore && [ "$rc" -eq 0 ]; then rc=1; fi; exit $rc' EXIT + +echo "== treatment: edit applied (${PATHS[*]}) ==" +run_eval treatment + +echo "== reverting edit for baseline ==" +git -C "$CLONE" stash push -q -- "${REL[@]}" +STASHED=1 + +echo "== baseline: edit reverted ==" +run_eval baseline + +restore # pop + re-sync now; the EXIT trap becomes a no-op + +EVAL="$EVAL" EXP="$EXP" OUT="$OUT" node -e ' +const path=require("path"); +const f=(p)=>{try{return require(path.resolve(p))}catch{return null}}; +const {EVAL,EXP,OUT}=process.env; +const b=f(`${OUT}/${EVAL}.baseline.json`), t=f(`${OUT}/${EVAL}.treatment.json`); +const chk=(r)=>{const c=(r&&r.checks)||[];return `${c.filter(x=>x&&x.passed).length}/${c.length}`}; +const row=(l,r)=>`${l.padEnd(10)} passed=${String(r&&r.passed).padEnd(5)} checks=${chk(r).padEnd(6)} docs.calls=${((r&&r.docs&&r.docs.calls)||[]).length}`; +console.log(`\n=== A/B: ${EVAL} (${EXP}) ===`); +console.log(row("baseline",b)); +console.log(row("treatment",t)); +const d=((t&&t.passed)?1:0)-((b&&b.passed)?1:0); +console.log(d>0?"\n-> edit IMPROVED the eval (FAIL->PASS)":d<0?"\n-> edit REGRESSED the eval (PASS->FAIL)":"\n-> no pass/fail change (compare checks / docs.calls above)"); +console.log(`saved: ${OUT}/${EVAL}.{baseline,treatment}.json`); +' diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh new file mode 100755 index 00000000..c1ad810f --- /dev/null +++ b/workspace/scripts/ab.test.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Zero-cost integration test for workspace/scripts/ab.sh. Fakes the eval via +# AB_EVAL_CMD (no model, no OpenAI, no services) and proves: the scoped +# stash/restore + result copy + comparison work, AND the selected edit and +# index are byte-identical after the run — including when a run fails mid-A/B +# (trap must restore). +# +# SAFETY: only ever touches a tracked skill file that is currently CLEAN (no +# working-tree or staged changes); it never checks out a file with your edits. +# If no clean skill file exists it fails with an actionable message rather +# than passing vacuously. Skills loop → sync is a no-op, so +# nothing else in the workspace is affected. Run: bash workspace/scripts/ab.test.sh +set -euo pipefail +cd "$(dirname "$0")/../.." + +SK=submodules/agent-skills +[ -e "$SK/.git" ] || { echo "ab.test: 0 checks ran — agent-skills submodule not initialized (run: mise run setup)" >&2; exit 1; } + +# pick a tracked SKILL.md with NO local changes, so restoring it can't lose work +REL="" +for c in $(git -C "$SK" ls-files 'skills/*/SKILL.md'); do + if git -C "$SK" diff --quiet -- "$c" && git -C "$SK" diff --cached --quiet -- "$c"; then REL="$c"; break; fi +done +[ -n "$REL" ] || { echo "ab.test: 0 checks ran — no CLEAN tracked skill file to test with" >&2; exit 1; } +FILE="$SK/$REL" +EVAL=ab-selftest; EXP=claude-sonnet-5 +RES="results/$EXP/$EVAL.json" + +# only now (after selecting a clean file) arm cleanup: revert our own edit, drop fixtures +MCP=submodules/mcp +MCPREL=packages/mcp-server-supabase/src/transports/stdio.ts +MCP_MUTATED=0 # cleanup may only revert $MCPREL if THIS test wrote to it — a + # user's own unstaged edit there skips the fixture block below, + # and must survive the run untouched. +cleanup() { + git -C "$SK" checkout -q -- "$REL" 2>/dev/null || true + [ "$MCP_MUTATED" = 1 ] && git -C "$MCP" checkout -q -- "$MCPREL" 2>/dev/null || true + rm -f "$RES" "results-ab/$EVAL".*.json +} +trap cleanup EXIT + +# fake eval: treatment passes, baseline fails; hard-fail baseline if AB_FAIL_BASELINE set. +FAKE='mkdir -p "$(dirname "$RES")"; if [ "$AB_LABEL" = baseline ] && [ -n "${AB_FAIL_BASELINE:-}" ]; then exit 7; fi; if [ "$AB_LABEL" = treatment ]; then p=true; else p=false; fi; printf "{\"passed\":%s,\"checks\":[{\"name\":\"x\",\"passed\":%s}],\"docs\":{\"calls\":[1,2]}}" "$p" "$p" > "$RES"' + +pass=0; fail=0 +ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $1 (want[$3] got[$2])"; fi; } + +# make an unstaged edit on the clean file +printf '\n\n' >> "$FILE" +edited=$(git -C "$SK" hash-object "$REL") +index0=$(git -C "$SK" diff --cached -- "$REL") # empty (was clean) + +# observable sync hook: every sync appends a line (treatment, baseline, restore = 3) +CNT=/tmp/ab_selftest_sync.cnt +SYNC="echo x >> $CNT" + +# --- happy path --- +: > "$CNT" +ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest.out 2>&1 \ + || { echo "FAIL: ab exited nonzero (happy path)"; sed 's/^/ /' /tmp/ab_selftest.out; fail=$((fail+1)); } +ck "edit preserved after success" "$(git -C "$SK" hash-object "$REL")" "$edited" +ck "index untouched after success" "$(git -C "$SK" diff --cached -- "$REL")" "$index0" +ck "treatment result copied" "$([ -f "results-ab/$EVAL.treatment.json" ] && echo y || echo n)" "y" +ck "baseline result copied" "$([ -f "results-ab/$EVAL.baseline.json" ] && echo y || echo n)" "y" +ck "reports IMPROVED (FAIL->PASS)" "$(grep -c 'IMPROVED' /tmp/ab_selftest.out)" "1" +ck "checks rendered (treatment 1/1)" "$(grep -c 'treatment.*checks=1/1' /tmp/ab_selftest.out)" "1" +ck "sync ran 3x (incl. post-restore)" "$(wc -l < "$CNT" | tr -d ' ')" "3" + +# --- failure path: baseline dies AFTER stash; ab's trap must restore the edit AND re-sync --- +: > "$CNT" +AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest2.out 2>&1 || true +ck "edit restored after mid-run failure" "$(git -C "$SK" hash-object "$REL")" "$edited" +ck "index untouched after failure" "$(git -C "$SK" diff --cached -- "$REL")" "$index0" +ck "trap re-synced after failure (3x)" "$(wc -l < "$CNT" | tr -d ' ')" "3" +ck "failure exit status preserved" "$(AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/dev/null 2>&1; echo $?)" "7" + +# --- restore-sync fails (3rd sync): must fail the run, keep the edit, skip the report --- +: > "$CNT" +SYNC_FAIL3="echo x >> $CNT; [ \$(wc -l < $CNT) -lt 3 ] || { echo sync3-boom >&2; exit 9; }" +rc3=$(ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC_FAIL3" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest3.out 2>&1; echo $?) +ck "restore-sync failure fails the run" "$([ "$rc3" -ne 0 ] && echo nonzero || echo zero)" "nonzero" +ck "edit intact after restore-sync failure" "$(git -C "$SK" hash-object "$REL")" "$edited" +ck "reports re-sync error" "$(grep -c 'post-restore re-sync failed' /tmp/ab_selftest3.out)" "1" +ck "no A/B report on failed restore" "$(grep -c '=== A/B' /tmp/ab_selftest3.out)" "0" + +# --- docs loop must reject a second path outside the content scope --- +rc4=$(AB_DRYRUN=1 bash workspace/scripts/ab.sh e x supabase/apps/docs/content/a.mdx supabase/apps/studio/foo.ts >/dev/null 2>&1; echo $?) +ck "rejects non-content path in docs loop" "$rc4" "2" + +# --- dirty clone index (e.g. intent-to-add residue) must be refused before stashing --- +echo probe > "$SK/.ab-selftest-idx-probe" +git -C "$SK" add -N .ab-selftest-idx-probe +rc5=$(ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest5.out 2>&1; echo $?) +git -C "$SK" reset -q -- .ab-selftest-idx-probe && rm -f "$SK/.ab-selftest-idx-probe" +ck "refuses dirty clone index" "$rc5" "1" +ck "explains staged-index refusal" "$(grep -c 'staged changes' /tmp/ab_selftest5.out)" "1" + +# --- patch-owned file A/B (the commit model's core win): an unstaged edit on a +# file the mcp enabler patch owns must A/B cleanly, and the plumbing marker +# commit must be untouched on both success and mid-run failure --- +if [ -e "$MCP/.git" ] && git -C "$MCP" diff --quiet -- "$MCPREL" && git -C "$MCP" diff --cached --quiet --ita-visible-in-index; then + marker0=$(git -C "$MCP" rev-parse HEAD) + MCP_MUTATED=1 + printf '\n// ab-selftest edit\n' >> "$MCP/$MCPREL" + edited2=$(git -C "$MCP" hash-object "$MCPREL") + ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$MCP/$MCPREL" >/tmp/ab_selftest6.out 2>&1 \ + || { echo "FAIL: patch-owned A/B exited nonzero"; sed 's/^/ /' /tmp/ab_selftest6.out; fail=$((fail+1)); } + ck "patch-owned: edit preserved" "$(git -C "$MCP" hash-object "$MCPREL")" "$edited2" + ck "patch-owned: marker commit intact" "$(git -C "$MCP" rev-parse HEAD)" "$marker0" + # the arm receipt must record the override the run actually used (mcp loop + # sets SUPABASE_MCP_SERVER_PATH); a null here mislabels the arm + ck "patch-owned: receipt records mcp override" \ + "$(node -p 'try{require("./results-ab/ab-selftest.treatment.json").provenance.mcp_override.path?"y":"n"}catch{"n"}')" "y" + AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$MCP/$MCPREL" >/dev/null 2>&1 || true + ck "patch-owned: edit restored after failure" "$(git -C "$MCP" hash-object "$MCPREL")" "$edited2" + ck "patch-owned: marker intact after failure" "$(git -C "$MCP" rev-parse HEAD)" "$marker0" + git -C "$MCP" checkout -q -- "$MCPREL" +else + echo "SKIP: mcp not initialized/clean — patch-owned A/B regression not run" +fi + +echo "ab.test: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/workspace/scripts/affected-task.sh b/workspace/scripts/affected-task.sh new file mode 100755 index 00000000..ffda3d53 --- /dev/null +++ b/workspace/scripts/affected-task.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Which evals does a change affect? Maps changed skill/docs/mcp paths to a +# ready-to-run eval command. e.g. mise run affected apps/docs/content/guides/auth/x.mdx +set -euo pipefail +cd "$(dirname "$0")/../.." +ROOT="$PWD" + +cd apps/framework && exec node --import tsx/esm "$ROOT/workspace/scripts/affected.ts" "$@" diff --git a/workspace/scripts/affected.ts b/workspace/scripts/affected.ts new file mode 100755 index 00000000..9512a5d7 --- /dev/null +++ b/workspace/scripts/affected.ts @@ -0,0 +1,233 @@ +#!/usr/bin/env tsx +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { basename, dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const ROOT = join(dirname(process.argv[1]), '..', '..'); +const requireFromFramework = createRequire( + join(ROOT, 'apps/framework/package.json') +); +const ALIASES: Record = { + functions: 'edge-functions', + rest: 'data-api', + api: 'data-api', +}; + +type Experiment = { name: string; skills: string[] }; +type Eval = { + id: string; + product: string[]; + topic: string[]; + interface?: string; +}; + +async function loadExperiments(): Promise { + const dir = join(ROOT, 'experiments'); + const experiments: Experiment[] = []; + + for (const file of readdirSync(dir) + .filter((file) => file.endsWith('.ts')) + .sort()) { + // Experiment files are runtime-selected plugins, matching run-eval.ts discovery. + const module = await import(pathToFileURL(join(dir, file)).href); + const config = module.default as { skills?: string[] }; + experiments.push({ + name: file.replace(/\.ts$/, ''), + skills: config.skills ?? [], + }); + } + + return experiments; +} + +async function discoverEvals(): Promise { + // Resolve from the repo root because this script intentionally lives outside that package. + const parserPath = requireFromFramework.resolve( + '@supabase-evals/core/eval-markdown' + ); + const { parseEvalMarkdown } = await import(pathToFileURL(parserPath).href); + const dir = join(ROOT, 'evals'); + if (!existsSync(dir)) return []; + + const evals: Eval[] = []; + for (const id of readdirSync(dir).sort()) { + const evalDir = join(dir, id); + if (!statSync(evalDir).isDirectory()) continue; + + const promptPath = join(evalDir, 'PROMPT.md'); + if (!existsSync(promptPath)) continue; // stray/partial dir: skip, don't abort the mapper + const metadata = parseEvalMarkdown( + readFileSync(promptPath, 'utf8'), + `evals/${id}/PROMPT.md` + ).metadata; + evals.push({ + id, + product: metadata.product, + topic: metadata.topic, + interface: metadata.interface, + }); + } + + return evals; +} + +function addPathTokens( + segments: string[], + vocabulary: Set, + destination: Set +): void { + for (const rawSegment of segments) { + const segment = rawSegment.replace(/\.[^.]+$/, ''); + const token = ALIASES[segment] ?? segment; + if (vocabulary.has(token)) destination.add(token); + } +} + +function matchingEvalIds(evals: Eval[], tokens: Set): string[] { + return evals + .filter((entry) => + [...entry.product, ...entry.topic].some((token) => tokens.has(token)) + ) + .map((entry) => entry.id); +} + +async function main(): Promise { + const paths = [...new Set(process.argv.slice(2))]; + if (paths.length === 0) { + console.log('Usage: affected.ts ...'); + return; + } + + const [experiments, evals] = await Promise.all([ + loadExperiments(), + discoverEvals(), + ]); + const vocabulary = new Set( + evals.flatMap((entry) => [...entry.product, ...entry.topic]) + ); + const skills = new Set(); + const docsTokens = new Set(); + const mcpTokens = new Set(); + const unknownPaths: string[] = []; + let allMcpEvals = false; + let serverWide = false; + + for (const rawPath of paths) { + const path = rawPath.replaceAll('\\', '/'); + let recognized = false; + + const skillMatch = path.match(/(?:^|\/)skills\/([^/]+)(?:\/|$)/); + if (skillMatch) { + skills.add(skillMatch[1]); + recognized = true; + } + + const docsMarker = 'apps/docs/content/'; + const docsIndex = path.indexOf(docsMarker); + if (docsIndex !== -1) { + addPathTokens( + path.slice(docsIndex + docsMarker.length).split('/'), + vocabulary, + docsTokens + ); + recognized = true; + } + + const isMcp = + path.includes('mcp-server-supabase/src/') || + path.includes('/tools/') || + path.startsWith('tools/') || + /^src\/(?:server\.ts|index\.ts|transports\/)/.test(path); + if (isMcp) { + recognized = true; + const file = basename(path); + const stem = file.replace(/\.ts$/, ''); + + if ( + file === 'server.ts' || + file === 'index.ts' || + /(?:^|\/)transports\//.test(path) + ) { + serverWide = true; + } else if (stem === 'docs-tools') { + allMcpEvals = true; + } else { + addPathTokens(stem.split('-'), vocabulary, mcpTokens); + } + } + + if (!recognized) unknownPaths.push(rawPath); + } + + if (unknownPaths.length > 0) { + console.log(`Ignored unknown paths: ${unknownPaths.join(', ')}`); + } + + let commandCount = 0; + + if (skills.size > 0) { + const names = experiments + .filter((experiment) => + experiment.skills.some((skill) => skills.has(skill)) + ) + .map((experiment) => experiment.name); + if (names.length > 0) { + console.log( + `mise run eval -- ${names.map((name) => `--experiment ${name}`).join(' ')} --suite regression` + ); + commandCount++; + } else { + console.log( + `No experiments use changed skills: ${[...skills].sort().join(', ')}` + ); + } + } + + if (docsTokens.size > 0) { + const ids = matchingEvalIds(evals, docsTokens); + if (ids.length > 0) { + // Docs impact is only measurable against the LOCAL index: the local mcp + // build (mcp-eval) pointed at the local content API. A bare `mise run + // eval` would query the production docs API and never see the edit. + // URL also lives in ab.sh / docs-api.sh / workspace README — keep in sync. + console.log('# needs: mise run docs-index && mise run docs-api'); + console.log( + `SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql mise run mcp-eval -- ${ids.map((id) => `--eval ${id}`).join(' ')}` + ); + commandCount++; + } + } + + if (mcpTokens.size > 0 || allMcpEvals) { + const ids = new Set(matchingEvalIds(evals, mcpTokens)); + if (allMcpEvals) { + for (const entry of evals) { + if (entry.interface === 'mcp') ids.add(entry.id); + } + } + if (ids.size > 0) { + // mcp-eval builds submodules/mcp and selects it via SUPABASE_MCP_SERVER_PATH; + // a bare `mise run eval` would run the published npx server instead. + console.log( + `mise run mcp-eval -- ${[...ids] + .sort() + .map((id) => `--eval ${id}`) + .join(' ')}` + ); + commandCount++; + } + } + + if (serverWide) { + console.log('mise run mcp-eval -- --smoke'); + commandCount++; + } + + if (commandCount === 0) console.log('No affected evals.'); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/workspace/scripts/apply-patches.sh b/workspace/scripts/apply-patches.sh new file mode 100755 index 00000000..f64e5b7e --- /dev/null +++ b/workspace/scripts/apply-patches.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# Materialize the enabler patches as identifiable LOCAL COMMITS in the +# supabase clone and the mcp submodule working tree: +# [eval-workspace-local] dev shim — must never leave this machine +# [eval-workspace-upstream] upstream candidate — leaves ONLY via +# `mise run publish … --with ` (reworded) +# Your own work sits ABOVE these as normal commits, so `git commit -am` can +# never sweep plumbing into it, and stash/A-B isolation stays clean. +# +# Idempotent: a patch whose marker commit already exists is skipped; patch +# content found uncommitted in the working tree (the old model) is migrated +# into a commit. Also installs a pre-push guard in the supabase clone and +# the mcp submodule (the two publishable repos) that blocks marker commits +# from being pushed anywhere. A repo whose dir is absent (supabase not +# cloned) or whose submodule is uninitialized (empty working tree, no .git) +# is skipped cleanly — never install anything on the host repo. +set -euo pipefail +cd "$(dirname "$0")/../.." +ROOT="$PWD" +source workspace/scripts/patches-lib.sh + +# Tree OID of " + ", built in a throwaway index. This is +# the ONE verification mechanism for "state matches the canonical patch": +# byte-exact tree identity, immune to the classic payload-diff blind spot +# (an identical +/- line reattributed to a different file). +patch_tree() { + local dir="$1" parent="$2" patch="$3" tmpidx out="" + tmpidx=$(mktemp /tmp/eval-workspace-idx-XXXXXX); rm -f "$tmpidx" # keep only the name: git must create the index itself + if GIT_INDEX_FILE="$tmpidx" git -C "$dir" read-tree "$parent" 2>/dev/null \ + && GIT_INDEX_FILE="$tmpidx" git -C "$dir" apply --cached "$ROOT/$patch" 2>/dev/null; then + out=$(GIT_INDEX_FILE="$tmpidx" git -C "$dir" write-tree 2>/dev/null || echo "") + fi + rm -f "$tmpidx" + echo "$out" +} + +apply_one() { + local dir="$1" patch="$2" subject sha h s + subject=$(patch_subject "$patch") + # marker commit already present? (search ALL local-only history) + sha="" + while read -r h s; do + [ "$s" = "$subject" ] && { sha="$h"; break; } + done </dev/null) +EOF + if [ -n "$sha" ]; then + # the .patch file is canonical — the commit must still match it EXACTLY: + # rebuild parent-tree + patch and compare tree OIDs + local want got + want=$(git -C "$dir" rev-parse "$sha^{tree}") + got=$(patch_tree "$dir" "$sha^" "$patch") + if [ "$got" = "$want" ]; then + echo " $(basename "$patch") already committed in $dir" + else + echo " ERROR: the $(basename "$patch") commit in $dir differs from the canonical patch file." >&2 + echo " refresh the file from the commit: git -C $dir diff $sha^ $sha > $patch" >&2 + echo " or drop the commit and re-apply: git -C $dir rebase --onto $sha^ $sha && mise run apply-patches" >&2 + exit 1 + fi + return + fi + # Build the commit from the CANONICAL patch via the index — never stage whole + # files, so user edits sitting in patch-owned files can't be absorbed. + if git -C "$dir" apply --index --check "$ROOT/$patch" 2>/dev/null; then + git -C "$dir" apply --index "$ROOT/$patch" # fresh clone: index + worktree + elif git -C "$dir" apply --cached --check "$ROOT/$patch" 2>/dev/null; then + git -C "$dir" apply --cached "$ROOT/$patch" # content already in worktree; extra edits stay unstaged above + else + echo " ERROR: $(basename "$patch") does not apply cleanly to $dir — regenerate it (workspace/patches/README.md)" >&2 + exit 1 + fi + # the staged state must be exactly parent-tree + canonical patch (tree identity) + if [ "$(git -C "$dir" write-tree)" != "$(patch_tree "$dir" HEAD "$patch")" ]; then + git -C "$dir" reset -q + echo " ERROR: staged state for $(basename "$patch") differs from the canonical patch — aborted (index reset)" >&2 + exit 1 + fi + GIT_AUTHOR_NAME=eval-workspace GIT_AUTHOR_EMAIL=eval-workspace@local \ + GIT_COMMITTER_NAME=eval-workspace GIT_COMMITTER_EMAIL=eval-workspace@local \ + git -C "$dir" commit -q -m "$subject" \ + -m "eval-workspace plumbing ($(patch_kind "$patch") kind), generated from $patch. Do not push; see workspace/patches/README.md." + echo " committed $(basename "$patch") -> $dir" +} + +install_pre_push_guard() { + local dir="$1" hooks hook + hooks=$(git -C "$dir" rev-parse --path-format=absolute --git-path hooks) # submodule .git is a file + mkdir -p "$hooks" + hook="$hooks/pre-push" + if [ -e "$hook" ] && ! grep -q 'eval-workspace pre-push guard' "$hook"; then + if [ -e "$hooks/pre-push.eval-workspace-chained" ] && ! cmp -s "$hook" "$hooks/pre-push.eval-workspace-chained"; then + echo " ERROR: $dir has BOTH a foreign pre-push hook AND a different previously-chained one — merge them manually:" >&2 + echo " foreign: $hook" >&2 + echo " chained: $hooks/pre-push.eval-workspace-chained" >&2 + exit 1 + fi + mv "$hook" "$hooks/pre-push.eval-workspace-chained" + echo " note: existing pre-push hook in $dir now chained after the eval-workspace guard" + fi + cat > "$hook" <<'HOOK' +#!/usr/bin/env bash +# eval-workspace pre-push guard (generated by scripts/apply-patches.sh). +# Blocks eval-workspace plumbing commits from leaving this machine. Publish work +# upstream via a clean cherry-picked branch: +# mise run publish [--with ] +# Deliberate override (skips ONLY the marker checks, never a chained hook): +# EVAL_WORKSPACE_ALLOW_PUSH=1 git push … +input=$(cat) +zero=0000000000000000000000000000000000000000 +if [ -z "${EVAL_WORKSPACE_ALLOW_PUSH:-}" ]; then + while read -r _lref lsha _rref _rsha; do + [ -z "$lsha" ] && continue + [ "$lsha" = "$zero" ] && continue # deleting a remote ref + # scan the FULL ancestry being pushed: markers only ever exist locally, so + # any marker anywhere below $lsha means plumbing would leave the machine. + # A failing log is a block too (fail-closed), never silently allowed. + if ! subjects=$(git log --format=%s "$lsha" 2>/dev/null); then + echo "push blocked: could not inspect the commits at $lsha (failing closed)." >&2 + exit 1 + fi + if printf '%s\n' "$subjects" | grep -q '^\[eval-workspace-local\]'; then + echo "push blocked: it contains [eval-workspace-local] plumbing commits (dev shims — never upstream)." >&2 + exit 1 + fi + if printf '%s\n' "$subjects" | grep -q '^\[eval-workspace-upstream\]'; then + echo "push blocked: it contains [eval-workspace-upstream] commits. Upstream those via a clean branch:" >&2 + echo " mise run publish --with " >&2 + exit 1 + fi + done <&2; exit 1; } + for p in $(patches_for "$repo"); do apply_one "$dir" "$p"; done +done +for name in $PATCH_REPOS; do + dir=$(repo_dir "$name") + [ -e "$dir/.git" ] && install_pre_push_guard "$dir" +done +echo " pre-push guards installed" diff --git a/workspace/scripts/clone-docs.sh b/workspace/scripts/clone-docs.sh new file mode 100755 index 00000000..70b2f6fd --- /dev/null +++ b/workspace/scripts/clone-docs.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/../.." +source workspace/scripts/patches-lib.sh +SUPABASE_REMOTE="${SUPABASE_REMOTE:-$(repo_remote supabase)}" + +if [ -e supabase/.git ]; then + echo "supabase/ already cloned; refreshing sparse checkout" +elif [ -e supabase ]; then + echo "ERROR: supabase/ exists but is not a Git clone" >&2 + exit 1 +else + git clone --filter=blob:none --no-checkout "$SUPABASE_REMOTE" supabase + git -C supabase sparse-checkout init --cone +fi + +git -C supabase sparse-checkout set \ + apps/docs \ + examples \ + packages/ai-commands packages/api-types packages/build-icons \ + packages/common packages/config packages/dev-tools \ + packages/eslint-config-supabase packages/icons packages/shared-data \ + packages/tsconfig packages/ui packages/ui-patterns \ + patches supabase + +git -C supabase checkout +pnpm --dir supabase install --filter docs... --frozen-lockfile diff --git a/workspace/scripts/docs-api.sh b/workspace/scripts/docs-api.sh new file mode 100755 index 00000000..aa1e82c1 --- /dev/null +++ b/workspace/scripts/docs-api.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Serve the docs content GraphQL API locally (standalone adapter) for search_docs. +# Point evals at it: SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql +set -euo pipefail +cd "$(dirname "$0")/../.." + +source workspace/scripts/load-keys.sh +set -a +source supabase/apps/docs/.env.development +set +a +eval "$(supabase status --workdir supabase -o env)" +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" +NODE_ENV=development \ +NEXT_PUBLIC_SUPABASE_URL="$API_URL" \ +NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" \ +OPENAI_API_KEY="$OPENAI_API_KEY" \ +exec pnpm --dir supabase/apps/docs exec tsx \ + --conditions=react-server \ + --tsconfig tsconfig.json \ + ../../../workspace/scripts/docs-content-api.ts diff --git a/workspace/scripts/docs-content-api.ts b/workspace/scripts/docs-content-api.ts new file mode 100644 index 00000000..5076b2e2 --- /dev/null +++ b/workspace/scripts/docs-content-api.ts @@ -0,0 +1,46 @@ +import { createServer } from 'node:http'; +import { + GET, + OPTIONS, + POST, +} from '../../supabase/apps/docs/app/api/graphql/route.ts'; + +const handlers = { GET, OPTIONS, POST }; +const port = Number(process.env.PORT ?? 3001); + +createServer(async (incoming, outgoing) => { + const url = new URL( + incoming.url ?? '/', + `http://${incoming.headers.host ?? `127.0.0.1:${port}`}` + ); + const handler = handlers[incoming.method as keyof typeof handlers]; + if (url.pathname !== '/docs/api/graphql' || !handler) { + outgoing.writeHead(404).end(); + return; + } + + const headers = new Headers(); + for (const [name, value] of Object.entries(incoming.headers)) { + if (Array.isArray(value)) + value.forEach((item) => headers.append(name, item)); + else if (value !== undefined) headers.set(name, value); + } + + const chunks: Buffer[] = []; + for await (const chunk of incoming) chunks.push(Buffer.from(chunk)); + const body = + incoming.method === 'GET' || incoming.method === 'HEAD' + ? undefined + : Buffer.concat(chunks).toString('utf8'); + const response = await handler( + new Request(url, { method: incoming.method, headers, body }) + ); + + outgoing.writeHead( + response.status, + Object.fromEntries(response.headers.entries()) + ); + outgoing.end(Buffer.from(await response.arrayBuffer())); +}).listen(port, '127.0.0.1', () => { + console.log(`Docs content API: http://127.0.0.1:${port}/docs/api/graphql`); +}); diff --git a/workspace/scripts/docs-down.sh b/workspace/scripts/docs-down.sh new file mode 100755 index 00000000..182dd86b --- /dev/null +++ b/workspace/scripts/docs-down.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/../.." +supabase stop --workdir supabase diff --git a/workspace/scripts/docs-embed-env.sh b/workspace/scripts/docs-embed-env.sh new file mode 100644 index 00000000..8eaf7b84 --- /dev/null +++ b/workspace/scripts/docs-embed-env.sh @@ -0,0 +1,22 @@ +# Shared preamble for the paid docs-embed paths (docs-index.sh, docs-seed.sh): +# fail-closed patch check, key loading, docs env, local-stack credentials, and +# the OpenAI preflight. Source from the repo root (the callers cd there first); +# not executable on its own. +if ! git -C supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then + echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 + exit 1 +fi + +source workspace/scripts/load-keys.sh +set -a +source supabase/apps/docs/.env.development +set +a +eval "$(supabase status --workdir supabase -o env)" +export NEXT_PUBLIC_SUPABASE_URL="$API_URL" +export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" +export SUPABASE_SECRET_KEY="$SECRET_KEY" +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" +workspace/scripts/openai-preflight.sh # fail fast if the key can't run embeddings +# Deliberately allow embedding without prod-only sources (e.g. lint-warnings, +# which needs DOCS_GITHUB_APP_*). Sources gate their own skip on this flag. +export DOCS_EMBED_ALLOW_MISSING_SOURCES=1 diff --git a/workspace/scripts/docs-index.sh b/workspace/scripts/docs-index.sh new file mode 100755 index 00000000..2ae557e6 --- /dev/null +++ b/workspace/scripts/docs-index.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/../.." +source workspace/scripts/docs-embed-env.sh + +pnpm --dir supabase/apps/docs run embeddings +node workspace/scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-seed.sh b/workspace/scripts/docs-seed.sh new file mode 100755 index 00000000..3be06de4 --- /dev/null +++ b/workspace/scripts/docs-seed.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/../.." +source workspace/scripts/docs-embed-env.sh + +printf '%s\n' \ + 'Full docs embedding rebuild' \ + 'Model: text-embedding-ada-002 (1536 dimensions)' \ + 'Rate: $0.10 per 1,000,000 input tokens; corpus cost is not known in advance.' +read -r -p "Type 'seed' to spend OpenAI credits and continue: " confirmation +if [ "$confirmation" != seed ]; then + echo 'Seed cancelled.' + exit 1 +fi + +pnpm --dir supabase/apps/docs run embeddings:refresh +node workspace/scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-up.sh b/workspace/scripts/docs-up.sh new file mode 100755 index 00000000..1971d65f --- /dev/null +++ b/workspace/scripts/docs-up.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/../.." +supabase start --workdir supabase \ + -x realtime,storage-api,imgproxy,mailpit,postgres-meta,studio,edge-runtime,logflare,vector,supavisor + +# Upstream page migrations grant service_role only Dxt (no CRUD). The new secret +# key authenticates as service_role and bypasses RLS but NOT SQL GRANTs, so the +# embedder (service_role) can't write the content tables. Grant CRUD locally +# (idempotent; native psql is absent on this host, so run it in the db container). +docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -q -c \ + "GRANT ALL ON public.page, public.page_section TO service_role; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO service_role; GRANT SELECT ON public.page, public.page_section TO anon, authenticated;" diff --git a/workspace/scripts/eval.sh b/workspace/scripts/eval.sh new file mode 100755 index 00000000..6f2cef7a --- /dev/null +++ b/workspace/scripts/eval.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Run evals with the workspace env applied. All args pass through to `pnpm eval`. +# API keys come from the macOS keychain if present (see README), else the repo-root .env. +# e.g. workspace/scripts/eval.sh --eval investigate-auth-001 --experiment claude-code-sonnet-5 +set -euo pipefail +cd "$(dirname "$0")/../.." + +[ -e .env ] || { echo ".env missing — run: mise run setup" >&2; exit 1; } +source workspace/scripts/load-keys.sh +pnpm eval "$@" diff --git a/workspace/scripts/hooks.test.sh b/workspace/scripts/hooks.test.sh new file mode 100755 index 00000000..4d6876ca --- /dev/null +++ b/workspace/scripts/hooks.test.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Self-test for the pre-push guard lifecycle (install / chain / reinstall). +# Exercises the guard in every publishable repo present locally — the +# supabase clone and the mcp submodule working tree. .git internals only, +# repo content untouched; every fixture is removed and the real guard +# reinstalled at the end. A repo that isn't cloned/initialized, or that +# already has a real chained hook, is skipped for that repo only. +set -euo pipefail +cd "$(dirname "$0")/../.." +source workspace/scripts/patches-lib.sh + +pass=0; fail=0 +ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $1 (want[$3] got[$2])"; fi; } + +CHAINED_PATHS=() +cleanup() { for c in ${CHAINED_PATHS[@]+"${CHAINED_PATHS[@]}"}; do rm -f "$c"; done; workspace/scripts/apply-patches.sh >/dev/null 2>&1 || true; } +trap cleanup EXIT + +test_guard_in() { + local repo="$1" dir hooks hook chained rc + dir=$(repo_dir "$repo") + [ -e "$dir/.git" ] || { echo "SKIP $repo: $dir not present"; return 0; } + hooks=$(git -C "$dir" rev-parse --path-format=absolute --git-path hooks) + hook="$hooks/pre-push" + chained="$hooks/pre-push.eval-workspace-chained" + [ ! -e "$chained" ] || { echo "SKIP $repo: $chained already exists (real chained hook — not touching it)"; return 0; } + CHAINED_PATHS+=("$chained") + + # baseline: generated guard present + workspace/scripts/apply-patches.sh >/dev/null + ck "$repo: guard installed" "$(grep -c 'eval-workspace pre-push guard' "$hook")" "1" + + # foreign hook appears -> next install chains it + printf '#!/usr/bin/env bash\n# foreign hook one\nexit 0\n' > "$hook"; chmod +x "$hook" + workspace/scripts/apply-patches.sh >/dev/null 2>&1 + ck "$repo: foreign hook chained" "$(grep -c 'foreign hook one' "$chained" 2>/dev/null || echo 0)" "1" + ck "$repo: guard reinstalled" "$(grep -c 'eval-workspace pre-push guard' "$hook")" "1" + + # a DIFFERENT foreign hook overwrites the generated guard -> reinstall must + # refuse rather than silently clobber the previously chained hook + printf '#!/usr/bin/env bash\n# foreign hook two\nexit 0\n' > "$hook"; chmod +x "$hook" + rc=$(workspace/scripts/apply-patches.sh >/dev/null 2>&1; echo $?) + ck "$repo: reinstall refuses to clobber chained" "$rc" "1" + ck "$repo: chained hook one preserved" "$(grep -c 'foreign hook one' "$chained")" "1" + ck "$repo: foreign hook two preserved" "$(grep -c 'foreign hook two' "$hook")" "1" + + # identical foreign hook (e.g. re-copied) -> harmless, install proceeds + cp "$chained" "$hook" + rc=$(workspace/scripts/apply-patches.sh >/dev/null 2>&1; echo $?) + ck "$repo: identical foreign re-chain ok" "$rc" "0" + ck "$repo: guard active again" "$(grep -c 'eval-workspace pre-push guard' "$hook")" "1" +} + +for r in $PATCH_REPOS; do + test_guard_in "$r" +done + +echo "hooks.test: $pass passed, $fail failed" +if [ "$pass" -eq 0 ] && [ "$fail" -eq 0 ]; then + echo "hooks.test: 0 checks ran (no patched repo present — run: mise run setup)" >&2 + exit 1 +fi +[ "$fail" -eq 0 ] diff --git a/workspace/scripts/load-keys.sh b/workspace/scripts/load-keys.sh new file mode 100755 index 00000000..bd73799d --- /dev/null +++ b/workspace/scripts/load-keys.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Export API keys from the macOS login keychain into the calling shell, so eval +# runs use them without a plaintext copy in .env. Keys not in the keychain are +# skipped; .env stays the fallback. Exported env beats node's --env-file, so a +# keychain value takes precedence when both exist. +# +# Usage: source scripts/load-keys.sh (the eval recipes do this) +# Running it directly is harmless — it only exports into its own child shell +# and prints nothing, so secrets never reach stdout/scrollback. +# Add a key: mise run store-key ANTHROPIC_API_KEY (hidden prompt; never the bare +# interactive `security -w` — it truncates pasted input at 128 chars) +# Check one: security find-generic-password -a "$USER" -s eval-workspace:OPENAI_API_KEY >/dev/null && echo present + +for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do + if value=$(security find-generic-password -a "$USER" -s "eval-workspace:$key" -w 2>/dev/null); then + export "$key=$value" + fi +done diff --git a/workspace/scripts/manifest.mjs b/workspace/scripts/manifest.mjs new file mode 100755 index 00000000..510c1df6 --- /dev/null +++ b/workspace/scripts/manifest.mjs @@ -0,0 +1,157 @@ +#!/usr/bin/env node +// Loader + CLI for manifest.json (lives at workspace/manifest.json, beside +// scripts/) — the single source of truth for the repos this workspace wires +// together: repo-root-relative checkout dirs, upstream remotes (clone +// entries only), and the ordered enabler-patch set per repo (patch names +// without the patches/ prefix or .patch suffix; "localPatches" marks the dev +// shims that must never reach an upstream PR — see patches/README.md). +// `kind: "submodule"` repos (mcp, skills) have no remote — .gitmodules owns +// that — and are never cloned by our scripts; a repo without `kind` is a +// plain clone and MUST have a remote. Every `dir` is repo-root-relative; +// resolving it is the caller's job (bash scripts `cd` to the repo root, +// two levels up from workspace/scripts, before using it). +// +// loadManifest() is THE loader: every consumer (the CLI below, the bash +// facade scripts/patches-lib.sh through it, and provenance.mjs via import) +// goes through the same read+validate, so a schema error fails identically +// everywhere. Never grep the JSON. +// +// CLI usage: +// manifest.mjs repos repo names, one per line, manifest order +// manifest.mjs get value; arrays print one item per line; +// absent optional keys print nothing (exit 0) +// manifest.mjs kind "local" | "upstream" for a patch name +import { readFileSync, realpathSync } from 'node:fs'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { dirname, join } from 'node:path'; + +const manifestPath = join( + dirname(fileURLToPath(import.meta.url)), + '..', + 'manifest.json' +); + +/** Read + validate manifest.json. Throws with an actionable message. */ +export function loadManifest() { + let manifest; + try { + manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + } catch (e) { + throw new Error(`cannot read ${manifestPath}: ${e.message}`); + } + + // --- schema validation: fail loud before answering any query --- + const isStr = (v) => typeof v === 'string' && v.length > 0; + const isStrArr = (v) => Array.isArray(v) && v.length > 0 && v.every(isStr); + if ( + typeof manifest?.repos !== 'object' || + manifest.repos === null || + Array.isArray(manifest.repos) + ) { + throw new Error('top-level "repos" object missing'); + } + for (const [name, repo] of Object.entries(manifest.repos)) { + if (!isStr(repo?.dir)) + throw new Error(`repos.${name}.dir must be a non-empty string`); + if (repo.kind !== undefined && repo.kind !== 'submodule') { + throw new Error(`repos.${name}.kind must be "submodule" when present`); + } + if (repo.kind === 'submodule') { + if (repo.remote !== undefined) + throw new Error( + `repos.${name}.remote must be absent for kind:"submodule" (.gitmodules owns the remote)` + ); + } else if (!isStr(repo.remote)) { + throw new Error( + `repos.${name}.remote must be a non-empty string (required for clone entries)` + ); + } + if (repo.patches !== undefined && !isStrArr(repo.patches)) + throw new Error( + `repos.${name}.patches must be a non-empty array of strings` + ); + if (repo.localPatches !== undefined) { + if (!isStrArr(repo.localPatches)) + throw new Error( + `repos.${name}.localPatches must be a non-empty array of strings` + ); + for (const p of repo.localPatches) { + if (!(repo.patches ?? []).includes(p)) + throw new Error( + `repos.${name}.localPatches has "${p}" which is not in patches` + ); + } + } + } + // Patch names must be globally unique: `kind` scans localPatches across ALL + // repos while other consumers scan per-repo — a duplicated name would let + // the two silently disagree. Also note "local" is opt-in: a patch absent + // from every localPatches defaults to upstream (publishable via --with). + const seen = new Set(); + for (const [name, repo] of Object.entries(manifest.repos)) { + for (const p of repo.patches ?? []) { + if (seen.has(p)) + throw new Error( + `patch name "${p}" appears under more than one repo (repos.${name}) — names must be globally unique` + ); + seen.add(p); + } + } + return manifest; +} + +// --- CLI: runs only when executed directly (not when imported) --- +// Compare REALPATHS: node resolves the ESM main-module URL through symlinks +// (macOS /tmp -> /private/tmp), so a naive pathToFileURL(argv[1]) mismatches. +const isMain = (() => { + try { + return ( + import.meta.url === + pathToFileURL(realpathSync(process.argv[1] ?? '')).href + ); + } catch { + return false; + } +})(); +if (isMain) { + const die = (msg) => { + console.error(`manifest.mjs: ${msg}`); + process.exit(1); + }; + + let manifest; + try { + manifest = loadManifest(); + } catch (e) { + die(e.message); + } + + const [cmd, ...args] = process.argv.slice(2); + switch (cmd) { + case 'repos': + console.log(Object.keys(manifest.repos).join('\n')); + break; + case 'get': { + const [name, key] = args; + const repo = manifest.repos[name ?? '']; + if (!repo) die(`unknown repo "${name ?? ''}"`); + if (!['dir', 'remote', 'patches', 'localPatches'].includes(key ?? '')) + die(`unknown key "${key ?? ''}"`); + const v = repo[key]; + if (v !== undefined) console.log(Array.isArray(v) ? v.join('\n') : v); + break; + } + case 'kind': { + const [patch] = args; + if (typeof patch !== 'string' || patch.length === 0) + die('kind: patch name required'); + const local = Object.values(manifest.repos).some((r) => + (r.localPatches ?? []).includes(patch) + ); + console.log(local ? 'local' : 'upstream'); + break; + } + default: + die(`unknown command "${cmd ?? ''}" (expected: repos | get | kind)`); + } +} diff --git a/workspace/scripts/mcp-eval.sh b/workspace/scripts/mcp-eval.sh new file mode 100755 index 00000000..98051f79 --- /dev/null +++ b/workspace/scripts/mcp-eval.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Run evals against the local mcp build (mise task `mcp-eval` builds it first). +# Add SUPABASE_CONTENT_API_URL= for a local docs index (Phase 2). +set -euo pipefail +cd "$(dirname "$0")/../.." + +SUPABASE_MCP_SERVER_PATH="submodules/mcp/packages/mcp-server-supabase" exec workspace/scripts/eval.sh "$@" diff --git a/workspace/scripts/openai-preflight.sh b/workspace/scripts/openai-preflight.sh new file mode 100755 index 00000000..654af55b --- /dev/null +++ b/workspace/scripts/openai-preflight.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Fail fast if OPENAI_API_KEY can't run embeddings, with a message specific to +# the failure kind (credential vs quota vs service vs network) so a transient +# outage doesn't send you into a needless key replacement. One ~1-token call, +# effectively free. Reads OPENAI_API_KEY from the environment. Used by +# docs-seed.sh and docs-index.sh. +set -euo pipefail +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + +code=$(curl -s -o /dev/null -w '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + https://api.openai.com/v1/embeddings \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"text-embedding-ada-002","input":"preflight"}') || code="000" + +case "$code" in + 200) ;; + 401|403) + echo "ERROR: OpenAI rejected the key (HTTP $code) — truncated / invalid / wrong scope." >&2 + echo 'Re-add the full key (hidden prompt, truncation-safe, any shell):' >&2 + echo ' mise run store-key OPENAI_API_KEY' >&2 + exit 1 ;; + 429) + echo "ERROR: OpenAI rate/quota limit (HTTP 429) — key is valid but rate-limited or out of quota. Retry later / check billing." >&2 + exit 1 ;; + 5??) + echo "ERROR: OpenAI service error (HTTP $code) — transient, retry shortly. The key is not necessarily bad." >&2 + exit 1 ;; + 000) + echo "ERROR: could not reach api.openai.com (network / DNS / timeout). Check connectivity; the key is not necessarily bad." >&2 + exit 1 ;; + *) + echo "ERROR: unexpected OpenAI response (HTTP $code)." >&2 + exit 1 ;; +esac diff --git a/workspace/scripts/patches-lib.sh b/workspace/scripts/patches-lib.sh new file mode 100644 index 00000000..0e491319 --- /dev/null +++ b/workspace/scripts/patches-lib.sh @@ -0,0 +1,54 @@ +# Bash facade over manifest.json — the single source of truth for repos, +# remotes, and enabler patches (order and kind; marker subjects derive from +# the kind). All data queries go through scripts/manifest.mjs, which validates +# the schema and fails loud. Sourced by apply-patches.sh, update.sh, +# publish.sh, status.sh, setup.sh, clone-docs.sh. Requires bash (BASH_SOURCE) +# and node (pinned via mise). +_MANIFEST_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +_manifest() { node "$_MANIFEST_LIB_DIR/manifest.mjs" "$@"; } + +# PATCH_REPOS = repos carrying enabler patches (manifest order): mcp, supabase. +# Patched repos are exactly the publishable ones today, so this is the one +# name; skills has no patches and is neither. Split only if the sets diverge. +# +# Fail-loud init: `for x in $(failing-cmd)` does NOT trip `set -e` in the +# sourcing script — the substitution failure is swallowed and the lists come +# out empty, turning a broken manifest into silent no-ops downstream +# (update.sh would "update" zero repos and exit 0). Capture first, verify. +_MANIFEST_REPOS=$(_manifest repos) || _MANIFEST_REPOS="" +if [ -z "$_MANIFEST_REPOS" ]; then + echo "patches-lib.sh: manifest.mjs returned no repos (broken manifest.json or node failure above) — aborting" >&2 + exit 1 +fi +PATCH_REPOS="" +for _r in $_MANIFEST_REPOS; do + if [ -n "$(_manifest get "$_r" patches)" ]; then + PATCH_REPOS="${PATCH_REPOS:+$PATCH_REPOS }$_r" + fi +done +unset _r _MANIFEST_REPOS + +repo_dir() { local d; d=$(_manifest get "$1" dir); echo "${d:-$1}"; } +repo_remote() { _manifest get "$1" remote; } + +patches_for() { + local out="" n + for n in $(_manifest get "$1" patches); do + out="${out:+$out }workspace/patches/$n.patch" + done + echo "$out" +} + +# local = dev shim, must NEVER reach an upstream PR +# upstream = real fix/feature; `mise run publish --with ` cherry-picks it +# onto a clean PR branch (reworded to drop the marker) +patch_kind() { _manifest kind "$(basename "$1" .patch)"; } + +patch_subject() { + local name; name=$(basename "$1" .patch) + if [ "$(patch_kind "$1")" = local ]; then + echo "[eval-workspace-local] $name" + else + echo "[eval-workspace-upstream] $name" + fi +} diff --git a/workspace/scripts/provenance.mjs b/workspace/scripts/provenance.mjs new file mode 100755 index 00000000..a759a4c8 --- /dev/null +++ b/workspace/scripts/provenance.mjs @@ -0,0 +1,346 @@ +#!/usr/bin/env node +// Workspace provenance receipt: exact repo SHAs, dirty state (tracked diff +// hash + untracked file content hashes), enabler-patch fingerprints + marker +// presence, and the docs-index stamp. This is the record of WHAT an eval +// actually ran against. +// +// Usage: +// provenance.mjs print the provenance object (JSON) +// provenance.mjs --embed add it as `provenance` to a result JSON +// provenance.mjs --stamp-docs-index write .docs-index-stamp.json describing +// the docs content state just embedded +// (called by docs-index.sh / docs-seed.sh +// after a SUCCESSFUL embed run only) +// +// Consumers: `status.sh --json` and ab.sh (embedded per-arm receipts, captured +// inside run_eval after each arm's sync so baseline/treatment receipts differ +// exactly by the edit under test). Report-only by design: nothing gates on +// these fields yet — receipt shape gets validated on real runs first. +import { + readFileSync, + writeFileSync, + existsSync, + rmSync, + renameSync, +} from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { loadManifest } from './manifest.mjs'; + +// This is the evals repo root (workspace/scripts -> ../..): supabase/, +// submodules/, and .docs-index-stamp.json all live here. The tracked-patches +// directory is a level lower, under workspace/ (see workspaceRoot below). +const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const workspaceRoot = join(root, 'workspace'); +// The one manifest loader (read + schema validation) — imported, not spawned, +// so this file and the manifest.mjs CLI structurally cannot diverge on how a +// bad manifest fails. +let manifest; +try { + manifest = loadManifest(); +} catch (e) { + console.error(`provenance.mjs: ${e.message}`); + process.exit(1); +} +const STAMP = join(root, '.docs-index-stamp.json'); +// Docs corpus source dir inside the supabase clone (the docs loop's edit scope). +const DOCS_CONTENT_DIR = 'apps/docs/content'; +// The embed pipeline's model/dims are read FROM the pipeline source at stamp +// time (no duplicated literals that can silently drift); extraction failure +// fails the stamp closed. +const EMBED_PIPELINE_FILE = + 'supabase/apps/docs/scripts/search/generate-embeddings.ts'; +const embedPipelineConfig = () => { + const src = readFileSync(join(root, EMBED_PIPELINE_FILE), 'utf8'); + const model = src.match(/EMBEDDING_MODEL:\s*'([^']+)'/)?.[1]; + const dims = src.match(/EMBEDDING_DIMENSION:\s*(\d+)/)?.[1]; + if (!model || !dims) { + throw new Error( + `cannot extract EMBEDDING_MODEL/EMBEDDING_DIMENSION from ${EMBED_PIPELINE_FILE} — pipeline layout changed?` + ); + } + return { model, dimensions: Number(dims) }; +}; + +const git = (dir, ...args) => { + try { + return execFileSync('git', ['-C', join(root, dir), ...args], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trimEnd(); + } catch { + return null; + } +}; +// Buffer-returning variant for anything that gets hashed (binary-safe). +const gitRaw = (dir, ...args) => { + try { + return execFileSync('git', ['-C', join(root, dir), ...args], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch { + return null; + } +}; +const sha256 = (data) => createHash('sha256').update(data).digest('hex'); + +// Untracked files are invisible to `git diff HEAD` but are real workspace +// state (e.g. a brand-new guide page) — record path + content hash, sorted. +// One implementation for both the whole-repo receipt and the docs-scoped +// stamp, so the two can never fingerprint untracked state differently. +const hashUntracked = (dir, ...pathspec) => + (git(dir, 'ls-files', '--others', '--exclude-standard', ...pathspec) ?? '') + .split('\n') + .filter(Boolean) + .sort() + .map((p) => { + try { + return { path: p, sha256: sha256(readFileSync(join(root, dir, p))) }; + } catch { + return { path: p, sha256: null }; + } + }); + +const repoState = (dir) => { + const dirty = (git(dir, 'status', '--porcelain') ?? '').length > 0; + // Hash the TRACKED diff only when there is one: an untracked-only dirty + // tree keeps dirty:true but dirty_diff_sha256:null (the untracked list + // below carries that state) — a hash-of-empty-diff here would mislead. + const trackedDiff = dirty ? gitRaw(dir, 'diff', 'HEAD', '--binary') : null; + const untracked = hashUntracked(dir); + return { + branch: git(dir, 'rev-parse', '--abbrev-ref', 'HEAD'), + sha: git(dir, 'rev-parse', 'HEAD'), + dirty, + dirty_diff_sha256: + trackedDiff && trackedDiff.length > 0 ? sha256(trackedDiff) : null, + untracked, + }; +}; + +// The two pinned submodules, keyed the way ab.sh/status.sh --json expect +// (basename of the submodules/ path). `git submodule status` prefixes an +// uninitialized entry with "-" — its SHA there is only the pinned index +// entry, not a real checkout, so that reads as null, not a fake SHA. +const submoduleShas = () => { + const shas = { 'agent-skills': null, mcp: null }; + for (const line of (git('.', 'submodule', 'status') ?? '').split('\n')) { + if (!line) continue; + const initialized = line[0] !== '-'; + const [sha, subPath] = line.slice(1).trim().split(/\s+/); + const name = subPath?.split('/').pop(); + if (name && name in shas) shas[name] = initialized ? sha : null; + } + return shas; +}; + +// Receipt assembly is LAZY: only the paths that emit a receipt (print/--embed) +// pay for it. --stamp-docs-index must not — repoState alone walks and hashes +// every clone's untracked files. +const buildProvenance = () => { + const hostState = repoState('.'); + const host = { + checkout_sha: hostState.sha, + checkout_dirty: hostState.dirty, + dirty_diff_sha256: hostState.dirty_diff_sha256, + untracked: hostState.untracked, + }; + const submodules = submoduleShas(); + + // Only non-submodule entries (today: supabase) get a full clone-shaped + // `repos.` record — the two submodules are already covered above by + // exact pinned SHA, which is what a submodule actually promises. + const repos = {}; + const patches = {}; + for (const [name, spec] of Object.entries(manifest.repos)) { + const initialized = existsSync(join(root, spec.dir, '.git')); + if (spec.kind !== 'submodule') { + repos[name] = initialized + ? { dir: spec.dir, cloned: true, ...repoState(spec.dir) } + : { dir: spec.dir, cloned: false }; + } + if (!spec.patches?.length || !initialized) continue; + + const localSubjects = ( + git(spec.dir, 'log', '--format=%s', 'HEAD', '--not', '--remotes') ?? '' + ).split('\n'); + for (const p of spec.patches) { + const file = join(workspaceRoot, 'patches', `${p}.patch`); + const kind = (spec.localPatches ?? []).includes(p) ? 'local' : 'upstream'; + patches[p] = { + repo: name, + kind, + patch_sha256: existsSync(file) ? sha256(readFileSync(file)) : null, + marker_present: localSubjects.includes(`[eval-workspace-${kind}] ${p}`), + }; + } + } + + // SUPABASE_MCP_SERVER_PATH swaps the mcp server for an arbitrary local + // build, so a receipt that only records the mcp submodule's pinned SHA + // would mislabel the arm. Record the override target verbatim plus, when + // it lives inside a git checkout (the mcp submodule, an out-of-tree build, + // anywhere), that checkout's exact HEAD and dirty state. + let mcp_override = null; + const overridePath = process.env.SUPABASE_MCP_SERVER_PATH; + if (overridePath) { + mcp_override = { + path: overridePath, + checkout_sha: null, + checkout_dirty: null, + }; + try { + // Same entrypoint shapes as the harness override accepts (.js/.mjs/.cjs). + const top = execFileSync( + 'git', + [ + '-C', + /\.[cm]?js$/.test(overridePath) + ? dirname(overridePath) + : overridePath, + 'rev-parse', + '--show-toplevel', + ], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] } + ).trimEnd(); + mcp_override.checkout_sha = execFileSync( + 'git', + ['-C', top, 'rev-parse', 'HEAD'], + { encoding: 'utf8' } + ).trimEnd(); + mcp_override.checkout_dirty = + execFileSync('git', ['-C', top, 'status', '--porcelain'], { + encoding: 'utf8', + }).trim().length > 0; + } catch { + // not inside a git checkout (or path missing): the verbatim path still records the arm + } + } + + let docs_index = null; + if (existsSync(STAMP)) { + try { + docs_index = JSON.parse(readFileSync(STAMP, 'utf8')); + } catch { + docs_index = { error: 'unreadable .docs-index-stamp.json' }; + } + } + + return { + generated_at: new Date().toISOString(), + host, + submodules, + repos, + patches, + mcp_override, + docs_index, + }; +}; + +// Combined fingerprint of the supabase enabler-patch set: part of the docs +// stamp identity, since the patches shape what the embed pipeline does. +const supabasePatchSetSha256 = () => { + const names = manifest.repos.supabase?.patches ?? []; + const h = createHash('sha256'); + for (const p of names) { + const file = join(workspaceRoot, 'patches', `${p}.patch`); + if (!existsSync(file)) { + throw new Error( + `manifest lists supabase patch "${p}" but ${file} is missing — manifest/patches drift` + ); + } + h.update(readFileSync(file)); + } + return h.digest('hex'); +}; + +const [cmd, target] = process.argv.slice(2); + +if (cmd === '--stamp-docs-index') { + // The stamp is a REPORT step running after the (paid) embed: a stamp + // failure must never turn a successful docs-index run into a nonzero exit. + // Fail closed on CONTENT (write no stamp, remove any stale one so nothing + // misdescribes the new embed) but exit 0 with a loud warning. + try { + if (!existsSync(join(root, 'supabase', '.git'))) { + throw new Error('--stamp-docs-index needs the supabase clone'); + } + const dirtyDiff = gitRaw( + 'supabase', + 'diff', + 'HEAD', + '--binary', + '--', + DOCS_CONTENT_DIR + ); + // Untracked corpus files (a brand-new guide) are part of the embedded state + // but invisible to `git diff HEAD` — scoped to the docs content slice. + const contentUntracked = hashUntracked('supabase', '--', DOCS_CONTENT_DIR); + const pipeline = embedPipelineConfig(); + const stamp = { + generated_at: new Date().toISOString(), + // SCOPE: this stamp identifies the REPO-BACKED docs content slice only + // (guides under apps/docs/content). The embed corpus also includes + // non-repo sources (GitHub discussions, generated reference docs, lint + // warnings, partner data) that this stamp cannot capture — and with + // DOCS_EMBED_ALLOW_MISSING_SOURCES, skipped sources leave their prior DB + // rows intact. Do NOT read this as identifying the whole index; that + // needs pipeline-side per-page checksum provenance (upstream fix first). + scope: 'repo-docs-content-only', + external_sources_not_captured: true, + supabase_sha: git('supabase', 'rev-parse', 'HEAD'), + repo_docs_content_state: { + content_tree: git('supabase', 'rev-parse', `HEAD:${DOCS_CONTENT_DIR}`), + content_dirty_diff_sha256: + dirtyDiff && dirtyDiff.length > 0 ? sha256(dirtyDiff) : null, + content_untracked: contentUntracked, + }, + embedding_model: pipeline.model, + embedding_dimensions: pipeline.dimensions, + patch_set_sha256: supabasePatchSetSha256(), + // schema_version deliberately omitted: there is no truthful source for it + // yet (the content-DB migration state isn't stamped by the pipeline). + }; + writeFileSync(STAMP, JSON.stringify(stamp, null, 2) + '\n'); + console.log(`stamped ${STAMP}`); + } catch (e) { + rmSync(STAMP, { force: true }); + console.error( + `provenance.mjs: WARNING — docs-index stamp NOT written (${e.message}); ` + + `any previous stamp was removed so receipts show docs_index: null instead of a stale claim. ` + + `The embed itself is unaffected.` + ); + } + process.exit(0); +} + +if (cmd === '--embed') { + if (!target) { + console.error('provenance.mjs: --embed needs a result-file path'); + process.exit(1); + } + let result; + try { + result = JSON.parse(readFileSync(target, 'utf8')); + } catch (e) { + console.error( + `provenance.mjs: --embed cannot read result JSON at ${target}: ${e.message}` + ); + process.exit(1); + } + result.provenance = buildProvenance(); + // Same-directory temp + rename: an interrupt mid-write cannot leave a + // truncated result file behind. + const tmp = `${target}.tmp-${process.pid}`; + writeFileSync(tmp, JSON.stringify(result, null, 2) + '\n'); + renameSync(tmp, target); +} else if (cmd === undefined) { + console.log(JSON.stringify(buildProvenance(), null, 2)); +} else { + console.error( + `provenance.mjs: unknown arg "${cmd}" (expected: nothing | --embed | --stamp-docs-index)` + ); + process.exit(1); +} diff --git a/workspace/scripts/publish.sh b/workspace/scripts/publish.sh new file mode 100755 index 00000000..6cf58f0c --- /dev/null +++ b/workspace/scripts/publish.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Build a CLEAN branch for an upstream PR, in a temporary worktree that starts +# at origin/ and contains ONLY what you select: +# - your own commits (everything above origin that isn't [eval-workspace-*]) +# - optionally, UPSTREAM-CANDIDATE plumbing commits via --with , +# reworded to drop the marker so the PR is clean +# [eval-workspace-local] commits can never be selected, and the pre-push guard +# blocks any marker commit from being pushed by accident. +# +# Usage: +# workspace/scripts/publish.sh --list what's publishable +# workspace/scripts/publish.sh [--with ]... +# : mcp (submodule) | supabase (clone) — the only publishable repos +set -euo pipefail +cd "$(dirname "$0")/../.." +ROOT="$PWD" +source workspace/scripts/patches-lib.sh + +repo="${1:-}"; shift || true +case " $PATCH_REPOS " in *" $repo "*) ;; *) + echo "usage: mise run publish [--with ]... | --list" >&2; exit 2 ;; +esac +dir=$(repo_dir "$repo") +[ -e "$dir/.git" ] || { echo "$dir not cloned" >&2; exit 1; } +branch=$(git -C "$dir" rev-parse --abbrev-ref HEAD) +[ "$branch" != HEAD ] || { echo "$dir is on a detached HEAD — check out a branch first" >&2; exit 1; } + +user_commits() { # oldest first, hash + subject, excluding plumbing markers + git -C "$dir" log --reverse --format='%H %s' "origin/$branch..HEAD" \ + | grep -v ' \[eval-workspace-' || true +} +candidate_sha() { # sha of an [eval-workspace-upstream] commit by patch name + git -C "$dir" log --format='%H %s' "origin/$branch..HEAD" \ + | grep -F "[eval-workspace-upstream] $1" | head -1 | cut -d' ' -f1 +} + +if [ "${1:-}" = "--list" ] || [ $# -eq 0 ]; then + echo "== $repo: publishable from $branch (base origin/$branch) ==" + echo "your commits:" + user_commits | sed 's/^\([0-9a-f]\{8\}\)[0-9a-f]*/ \1/' || true + [ -n "$(user_commits)" ] || echo " (none)" + if [ -n "$(patches_for "$repo")" ]; then + echo "upstream-candidate plumbing (add with --with ):" + for p in $(patches_for "$repo"); do + [ "$(patch_kind "$p")" = upstream ] || continue + echo " $(basename "$p" .patch)" + done + fi + echo + echo "then: mise run publish $repo (--all | --commit ...) [--with ]..." + exit 0 +fi + +topic="$1"; shift +case "$topic" in --*) echo "topic branch must come before the selection flags" >&2; exit 2 ;; esac +ALL=0; COMMITS=(); WITH=() +while [ $# -gt 0 ]; do + case "$1" in + --all) ALL=1; shift ;; + --commit) shift; [ -n "${1:-}" ] || { echo "--commit needs a sha" >&2; exit 2; }; COMMITS+=("$1"); shift ;; + --with) shift; [ -n "${1:-}" ] || { echo "--with needs a patch name" >&2; exit 2; }; WITH+=("$1"); shift ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +# resolve selections -> SHAs. Commit selection is EXPLICIT: --all takes every +# commit of yours; --commit picks specific ones — so stacked, unrelated +# experiments can't ride along into a PR by accident. +PICKS=() +for w in ${WITH[@]+"${WITH[@]}"}; do + sha=$(candidate_sha "$w") + [ -n "$sha" ] || { echo "no [eval-workspace-upstream] commit for '$w' in $dir (see: mise run publish $repo --list)" >&2; exit 1; } + PICKS+=("$sha") +done +if [ "$ALL" = 1 ]; then + [ ${#COMMITS[@]} -eq 0 ] 2>/dev/null || { echo "use --all or --commit, not both" >&2; exit 2; } + while read -r sha _; do [ -n "$sha" ] && PICKS+=("$sha"); done </dev/null; then + for c in ${COMMITS[@]+"${COMMITS[@]}"}; do + sha=$(git -C "$dir" rev-parse --verify --quiet "$c^{commit}") \ + || { echo "not a commit in $dir: $c" >&2; exit 1; } + user_commits | grep -q "^$sha " \ + || { echo "$c is not one of YOUR commits above origin/$branch (see: mise run publish $repo --list)" >&2; exit 1; } + PICKS+=("$sha") + done +fi +if [ ${#PICKS[@]} -eq 0 ] 2>/dev/null; then + echo "nothing selected. Pick commits explicitly:" >&2 + echo " mise run publish $repo $topic --all # every commit of yours" >&2 + echo " mise run publish $repo $topic --commit ... # specific commits" >&2 + echo " mise run publish $repo $topic --with # an upstream-candidate enabler" >&2 + echo >&2 + echo "your commits (mise run publish $repo --list):" >&2 + user_commits | sed 's/^/ /' >&2 + exit 2 +fi +# chronological order (parents first) +ORDERED=() +while read -r sha; do + for p in ${PICKS[@]+"${PICKS[@]}"}; do [ "$sha" = "$p" ] && ORDERED+=("$sha"); done +done <&2; exit 1; } +git -C "$dir" worktree add -q -b "$topic" "$wt" "origin/$branch" \ + || { echo "could not create worktree/branch '$topic' (does the branch already exist?)" >&2; exit 1; } + +fail() { + git -C "$wt" cherry-pick --abort 2>/dev/null || true + git -C "$dir" worktree remove --force "$wt" 2>/dev/null || true + git -C "$dir" branch -D "$topic" 2>/dev/null || true + echo "publish aborted: $1" >&2 + exit 1 +} +for sha in ${ORDERED[@]+"${ORDERED[@]}"}; do + git -C "$wt" cherry-pick "$sha" >/dev/null || fail "cherry-pick conflict at $(git -C "$dir" log -1 --format='%h %s' "$sha")" + subj=$(git -C "$wt" log -1 --format=%s) + case "$subj" in + "[eval-workspace-upstream] "*) # reword: drop the marker + plumbing boilerplate + git -C "$wt" log -1 --format=%B \ + | sed -e '1s/^\[eval-workspace-upstream\] //' -e '/eval-workspace plumbing/d' \ + | git -C "$wt" commit -q --amend -F - ;; + esac +done + +echo "== clean PR branch ready ==" +git -C "$wt" log --oneline "origin/$branch..HEAD" | sed 's/^/ /' +echo +echo "Review and push:" +echo " cd $wt" +echo " git push origin HEAD # then open the PR" +echo +echo "When merged (or abandoned), clean up:" +echo " git -C $dir worktree remove $wt && git -C $dir branch -D $topic" diff --git a/workspace/scripts/setup.sh b/workspace/scripts/setup.sh new file mode 100755 index 00000000..d5e5a987 --- /dev/null +++ b/workspace/scripts/setup.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Install deps, init submodules (agent-skills + mcp), apply patches, wire +# .env, print status. Idempotent: safe to re-run any time. +set -euo pipefail + +cd "$(dirname "$0")/../.." + +log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33mwarn:\033[0m %s\n' "$*" >&2; } + +# --- required tooling --- +for bin in git pnpm; do + command -v "$bin" >/dev/null || { warn "missing required tool: $bin"; exit 1; } +done +if command -v docker >/dev/null; then + docker info >/dev/null 2>&1 || warn "docker installed but not running — start it before sandbox-runtime evals." +else + warn "docker not found — sandbox-runtime evals need it (tools-mode evals don't)." +fi + +# --- deps --- +if [ -d node_modules ]; then + log "node_modules present — skipping install." +else + log "Installing deps (corepack + pnpm install)…" + corepack enable >/dev/null 2>&1 || true + pnpm install +fi + +# --- submodules: agent-skills + mcp (supabase clone stays opt-in) --- +log "Syncing submodules (agent-skills, mcp)…" +git submodule update --init submodules/agent-skills submodules/mcp + +# --- enabler plumbing: marker commits + pre-push guards (idempotent) --- +workspace/scripts/apply-patches.sh + +# --- .env --- +if [ -f .env ]; then + log ".env already exists — leaving it." +else + cp .env.example .env + log "Created .env from .env.example — fill in your API keys." +fi +if [ "$(uname -s)" = Darwin ]; then + log "Tip: store keys in the keychain instead of .env — mise run store-key " +fi + +log "Setup done. (Docs loop is optional and heavy — see: mise run clone-docs)" +echo +workspace/scripts/status.sh diff --git a/workspace/scripts/status.sh b/workspace/scripts/status.sh new file mode 100755 index 00000000..53d56f4b --- /dev/null +++ b/workspace/scripts/status.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Workspace status: host repo + submodule + clone state, env keys present, tooling. +set -euo pipefail + +cd "$(dirname "$0")/../.." + +# --json: machine-readable provenance receipt (host, submodules, repos, patches, docs stamp) +if [ "${1:-}" = "--json" ]; then exec node workspace/scripts/provenance.mjs; fi +source workspace/scripts/patches-lib.sh + +# label, dir, [missing-message]. `.git` is a dir for clones and a file for +# submodules, so -e covers both. +repo_status() { + local label="$1" dir="$2" missing="${3:-not cloned}" branch sha dirty + if [ ! -e "$dir/.git" ]; then + printf ' %-22s %s\n' "$label" "$missing" + return + fi + branch=$(git -C "$dir" rev-parse --abbrev-ref HEAD 2>/dev/null || echo '?') + sha=$(git -C "$dir" rev-parse --short HEAD 2>/dev/null || echo '?') + if [ -n "$(git -C "$dir" status --porcelain 2>/dev/null)" ]; then dirty=dirty; else dirty=clean; fi + printf ' %-22s %-22s %-10s %s\n' "$label" "$branch" "$sha" "$dirty" +} + +NEXT="" +echo "Repos:" +# The host repo is this checkout itself (evals); agent-skills and mcp are its +# two pinned submodules (hand-nested rows); supabase is the only opt-in clone. +repo_status "evals (host)" "." +repo_status " - agent-skills" "$(repo_dir skills)" "not initialized" +repo_status " - mcp" "$(repo_dir mcp)" "not initialized" +for _repo in $PATCH_REPOS; do + case "$_repo" in skills|mcp) continue ;; esac + repo_status "$_repo" "$(repo_dir "$_repo")" +done +unset _repo + +# Bootstrap order (mirrors mise run setup): deps, then submodules, then +# enabler plumbing, then keys. The supabase clone is opt-in for the docs loop +# only — its absence is informational (shown above) and never gates Ready. +[ -d node_modules ] || NEXT="${NEXT} pnpm install # install workspace deps\n" +if [ ! -e "$(repo_dir skills)/.git" ] || [ ! -e "$(repo_dir mcp)/.git" ]; then + NEXT="${NEXT} git submodule update --init submodules/agent-skills submodules/mcp # init submodules\n" +fi +# enabler plumbing present? (marker commits; see apply-patches.sh) — only +# checked for repos that are actually present (supabase clone, mcp submodule). +for _r in $PATCH_REPOS; do + _d=$(repo_dir "$_r") + [ -e "$_d/.git" ] || continue + _subjects=$(git -C "$_d" log --format=%s HEAD --not --remotes 2>/dev/null || true) + for _p in $(patches_for "$_r"); do + if ! printf '%s\n' "$_subjects" | grep -qxF "$(patch_subject "$_p")"; then + NEXT="${NEXT} mise run apply-patches # enabler plumbing commit(s) missing in $_d\n" + break + fi + done +done +# pre-push guards active? (apply-patches installs them) +for _n in $PATCH_REPOS; do + _d=$(repo_dir "$_n") + [ -e "$_d/.git" ] || continue + _hooks=$(git -C "$_d" rev-parse --path-format=absolute --git-path hooks 2>/dev/null || true) + if [ -n "$_hooks" ] && ! grep -q 'eval-workspace pre-push guard' "$_hooks/pre-push" 2>/dev/null; then + NEXT="${NEXT} mise run apply-patches # pre-push guard missing in $_d\n" + fi +done +unset _r _n _d _subjects _p _hooks + +echo +echo "Credentials (source per key; ANTHROPIC + OPENAI required, GEMINI optional):" +IS_DARWIN=""; [ "$(uname -s)" = Darwin ] && IS_DARWIN=1 +for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do + src="" + [ -n "$IS_DARWIN" ] && security find-generic-password -a "$USER" -s "eval-workspace:$key" >/dev/null 2>&1 && src="keychain" + if [ -f .env ] && grep -qE "^${key}=.+" .env; then src="${src:+$src, }.env"; fi + printf ' %-24s %s\n' "$key" "${src:-MISSING}" + if [ -z "$src" ] && [ "$key" != GEMINI_API_KEY ]; then + if [ -n "$IS_DARWIN" ]; then + NEXT="${NEXT} mise run store-key $key\n" + else + NEXT="${NEXT} add ${key}=… to .env # git-ignored; no macOS keychain here (see README \"Credentials\")\n" + fi + fi +done + +echo +echo "Tooling:" +for bin in mise pnpm docker node; do + if command -v "$bin" >/dev/null; then printf ' %-7s %s\n' "$bin" "$(command -v "$bin")"; else printf ' %-7s missing\n' "$bin"; fi +done +if command -v docker >/dev/null; then + if docker info >/dev/null 2>&1; then echo " docker (running)"; else echo " docker (not running)"; fi +fi + +echo +if [ -n "$NEXT" ]; then + echo "Next (in order):" + printf "%b" "$NEXT" | awk '!seen[$0]++' +else + echo "Ready. Try:" + echo " mise run ab-test # zero-cost self-check of the A/B runner" + echo " mise run ab # A/B readiness probe (head-to-head evals)" + echo " mise run eval -- --eval --experiment # run an eval (see evals/)" +fi diff --git a/workspace/scripts/status.test.sh b/workspace/scripts/status.test.sh new file mode 100755 index 00000000..18124cb0 --- /dev/null +++ b/workspace/scripts/status.test.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Self-test for status.sh's Next/Ready diagnosis. Runs the WORKING-TREE script +# against synthetic workspace states in a temp dir (never touches this one). +# USER=nobody forces keychain misses so results don't depend on real keys. +set -euo pipefail +cd "$(dirname "$0")/../.." +SRC="$PWD" + +pass=0; fail=0 +ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $1 (want[$3] got[$2])"; fi; } + +run_case() { # $1 = label, $2 = state-builder function; output -> /tmp/status_case.out + local tmp; tmp="$(mktemp -d /tmp/eval-ws-status-XXXXXX)" + mkdir -p "$tmp/workspace/scripts" "$tmp/bin" + cp "$SRC/workspace/scripts/status.sh" "$SRC/workspace/scripts/patches-lib.sh" "$SRC/workspace/scripts/manifest.mjs" "$tmp/workspace/scripts/" + cp "$SRC/workspace/manifest.json" "$tmp/workspace/" + cp "$SRC/.env.example" "$tmp/" + # This checkout IS the host repo in real usage — fake the marker so the + # host row never misreports "not cloned" while a case exercises other gaps. + mkdir -p "$tmp/.git" + # OS-deterministic: default to Darwin regardless of host; a case may overwrite bin/uname. + printf '#!/bin/sh\necho Darwin\n' > "$tmp/bin/uname"; chmod +x "$tmp/bin/uname" + ( cd "$tmp" && "$2" ) + ( cd "$tmp" && USER=nobody PATH="$PWD/bin:$PATH" bash workspace/scripts/status.sh 2>/dev/null ) > /tmp/status_case.out + rm -rf "$tmp" +} +has() { grep -qF -- "$1" /tmp/status_case.out && echo y || echo n; } + +# --- fresh: nothing set up --- +state_fresh() { :; } +run_case fresh state_fresh +ck "fresh: suggests pnpm install" "$(has 'pnpm install')" y +ck "fresh: suggests submodule init" "$(has 'submodule update --init')" y +ck "fresh: suggests required keys" "$(has 'mise run store-key OPENAI_API_KEY')" y +ck "fresh: not Ready" "$(has 'Ready. Try:')" n + +# --- deps installed, submodules not yet initialized --- +state_deps_only() { mkdir -p node_modules; cp .env.example .env; } +run_case deps-only state_deps_only +ck "deps-only: no pnpm-install suggestion" "$(has 'pnpm install')" n +ck "deps-only: suggests submodule init" "$(has 'submodule update --init')" y +ck "deps-only: not Ready" "$(has 'Ready. Try:')" n + +# --- submodules initialized (markers/hooks still missing), env missing --- +state_submodules_init() { mkdir -p node_modules submodules/agent-skills/.git submodules/mcp/.git; } +run_case submodules-init state_submodules_init +ck "submodules-init: no pnpm-install suggestion" "$(has 'pnpm install')" n +ck "submodules-init: no submodule-init suggestion" "$(has 'submodule update --init')" n +ck "submodules-init: suggests required keys" "$(has 'mise run store-key ANTHROPIC_API_KEY')" y +ck "submodules-init: not Ready" "$(has 'Ready. Try:')" n +# supabase was never cloned in this state — its absence must stay informational, +# never surfacing an action item (there is no clone-docs step in the bootstrap order). +ck "submodules-init: supabase absence isn't an action item" "$(has 'clone-docs')" n + +# --- complete workspace state (deps + submodules + env), only plumbing/keys stay open --- +state_keys_only() { mkdir -p node_modules submodules/agent-skills/.git submodules/mcp/.git; cp .env.example .env; } +run_case keys-only state_keys_only +ck "keys-only: no pnpm-install suggestion" "$(has 'pnpm install')" n +ck "keys-only: no submodule-init suggestion" "$(has 'submodule update --init')" n +ck "keys-only: suggests keys" "$(has 'mise run store-key ANTHROPIC_API_KEY')" y +ck "keys-only: not Ready" "$(has 'Ready. Try:')" n + +# --- non-Darwin: same complete-but-keyless state; keys must route to .env, never store-key --- +state_linux_keys() { state_keys_only; printf '#!/bin/sh\necho Linux\n' > bin/uname; } +run_case linux-keys state_linux_keys +ck "linux: no store-key suggestion" "$(has 'mise run store-key')" n +ck "linux: routes keys to .env" "$(has 'add ANTHROPIC_API_KEY=')" y +ck "linux: not Ready" "$(has 'Ready. Try:')" n + +echo "status.test: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/workspace/scripts/store-key.sh b/workspace/scripts/store-key.sh new file mode 100755 index 00000000..535a9ae6 --- /dev/null +++ b/workspace/scripts/store-key.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Store an API key in the macOS keychain (item eval-workspace:), reading the +# value with a hidden prompt. Run via `mise run store-key ` — the bash +# shebang makes it work no matter the caller's shell (zsh's `read -p` differs). +# +# Why not the interactive `security -w` prompt: it silently truncates pasted +# input at 128 chars, producing invalid keys. Reading into a variable and +# passing it as an argument is immune, and the value never reaches shell +# history or stdout. +set -euo pipefail + +KEY="${1:-}" +case "$KEY" in + ANTHROPIC_API_KEY|OPENAI_API_KEY|GEMINI_API_KEY) ;; + *) echo "usage: mise run store-key " >&2; exit 2 ;; +esac + +if [ "$(uname -s)" != Darwin ]; then + echo "store-key uses the macOS keychain — not available on this platform." >&2 + echo "Add ${KEY}= to the git-ignored .env instead (see README \"Credentials\")." >&2 + exit 2 +fi + +read -rsp "$KEY (input hidden): " value; echo +[ -n "$value" ] || { echo "empty value — nothing stored" >&2; exit 1; } +len=${#value} +security add-generic-password -U -a "$USER" -s "eval-workspace:$KEY" -w "$value" +unset value +echo "stored eval-workspace:$KEY ($len chars)" diff --git a/workspace/scripts/update.sh b/workspace/scripts/update.sh new file mode 100755 index 00000000..c4514e57 --- /dev/null +++ b/workspace/scripts/update.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Update the supabase clone: fetch upstream and REBASE local work — the +# [eval-workspace-*] plumbing commits plus any commits of yours — onto the +# new tip. Uncommitted edits survive via --autostash. A rebase conflict means +# upstream drifted under a plumbing commit (regenerate that patch; see +# workspace/patches/README.md) or under your own commits; the rebase is +# aborted so the workspace is left as found. +# +# mcp and skills are pin-driven submodules (no remote in manifest.json) — +# not managed here. Bump the pin with `git submodule update --remote ` +# and record it via the M3 pin flow, then re-run `mise run apply-patches`. +# +# Usage: workspace/scripts/update.sh [--check] [repo...] +# --check fetch + report how far behind each clone is; changes nothing +# repos default: every patch-carrying repo from manifest.json +set -euo pipefail +cd "$(dirname "$0")/../.." +source workspace/scripts/patches-lib.sh + +CHECK=0 +[ "${1:-}" = "--check" ] && { CHECK=1; shift; } +REPOS=("$@") +[ ${#REPOS[@]} -gt 0 ] 2>/dev/null || REPOS=($PATCH_REPOS) + +FAILED=0 +for repo in ${REPOS[@]+"${REPOS[@]}"}; do + dir=$(repo_dir "$repo") + remote=$(repo_remote "$repo") + if [ -z "$remote" ]; then + echo "== $repo: pin-driven ($dir) — not fetched/rebased here." + echo " bump: git submodule update --remote $dir (then record the pin — see M3 pin flow)" + continue + fi + if [ ! -e "$dir/.git" ]; then echo "== $repo: not cloned — skipping"; continue; fi + branch=$(git -C "$dir" rev-parse --abbrev-ref HEAD) + old_upstream=$(git -C "$dir" rev-parse "origin/$branch" 2>/dev/null || echo "") + git -C "$dir" fetch -q + behind=$(git -C "$dir" rev-list --count "HEAD..origin/$branch" 2>/dev/null || echo '?') + + if [ "$CHECK" = 1 ]; then + printf '== %-9s %s@%s is %s commit(s) behind origin/%s\n' "$repo:" "$branch" "$(git -C "$dir" rev-parse --short HEAD)" "$behind" "$branch" + continue + fi + + echo "== $repo: $branch, $behind commit(s) behind — updating" + git -C "$dir" diff --cached --quiet --ita-visible-in-index \ + || { echo " $dir index has staged changes — unstage or commit first" >&2; FAILED=1; continue; } + + if [ "$behind" = 0 ]; then + echo " already up to date" + continue + fi + + if ! git -C "$dir" rebase --autostash -q "origin/$branch"; then + git -C "$dir" rebase --abort 2>/dev/null || true + echo " REBASE CONFLICT: upstream drifted under a plumbing commit or your work — nothing changed." >&2 + echo " Fix: regenerate the conflicting patch (workspace/patches/README.md) or rebase manually in $dir/." >&2 + FAILED=1 + continue + fi + echo " now at $(git -C "$dir" rev-parse --short HEAD) (upstream $(git -C "$dir" rev-parse --short "origin/$branch"))" + + new_upstream=$(git -C "$dir" rev-parse "origin/$branch") + if [ "$old_upstream" != "$new_upstream" ]; then + case "$repo" in + supabase) + if ! git -C "$dir" diff --quiet "$old_upstream" "$new_upstream" -- pnpm-lock.yaml; then + echo " lockfile changed — pnpm install (docs filter)"; pnpm --dir "$dir" install --filter docs... --silent + fi + if [ -n "$(git -C "$dir" diff --name-only "$old_upstream" "$new_upstream" -- apps/docs/content | head -1)" ]; then + echo " note: docs content changed upstream — the next docs-index re-embeds changed pages (costs cents)" + fi ;; + esac + fi +done + +[ "$FAILED" = 0 ] || { echo; echo "update finished with errors (see above)"; exit 1; }