From fb031b95f282b5f0bb47a1e9a1001a5ca6a34d5b Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 09:31:50 +0200 Subject: [PATCH 1/9] feat: thread a docs Content API override to the MCP server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SUPABASE_CONTENT_API_URL (or the contentApiUrl option) adds --content-api-url to the server args, pointing search_docs at an alternative docs index (e.g. a locally built one). Requires a server that understands the flag — the SUPABASE_MCP_SERVER_PATH build; inert otherwise. Retires the eval-workspace enabler patch of the same name. --- packages/core/src/index.ts | 7 +++++++ packages/core/src/mcp-server.test.ts | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2222d211..f0ad75a5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -897,6 +897,7 @@ export function supabaseMcpServer( options: { features?: string[]; version?: string; + contentApiUrl?: string; } = {}, ): McpServerDefinition { const features = options.features ?? [ @@ -927,6 +928,12 @@ export function supabaseMcpServer( // platform-independent (it queries the public docs GraphQL API), so a // docs-only server runs standalone with no `--api-url`. if (apiUrl) serverArgs.push("--api-url", apiUrl); + // Alternative docs Content API endpoint (e.g. a locally built docs + // index). Requires a server that understands --content-api-url — the + // SUPABASE_MCP_SERVER_PATH build; only set the env var alongside it. + const contentApiUrl = + options.contentApiUrl ?? process.env.SUPABASE_CONTENT_API_URL; + if (contentApiUrl) serverArgs.push("--content-api-url", contentApiUrl); const local = resolveLocalMcpServer(); if (local) { diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts index b85b75c8..5bfefb68 100644 --- a/packages/core/src/mcp-server.test.ts +++ b/packages/core/src/mcp-server.test.ts @@ -12,6 +12,7 @@ import { // Stub (not mutate) env so pre-existing SUPABASE_* values are restored per test. function clearEnv() { vi.stubEnv("SUPABASE_MCP_SERVER_PATH", undefined); + vi.stubEnv("SUPABASE_CONTENT_API_URL", undefined); } // A real on-disk build layout: the override path is existence-checked, so the @@ -42,6 +43,26 @@ describe("supabaseMcpServer().createConfig", () => { `@supabase/mcp-server-supabase@${MCP_SERVER_VERSION}`, ); expect(config.args).toContain("--api-url"); + expect(config.args).not.toContain("--content-api-url"); + }); + + it("threads --content-api-url from the env var", async () => { + clearEnv(); + vi.stubEnv("SUPABASE_CONTENT_API_URL", "https://env.test/gql"); + const { config } = await supabaseMcpServer().createConfig({}); + const i = config.args.indexOf("--content-api-url"); + expect(i).toBeGreaterThan(-1); + expect(config.args[i + 1]).toBe("https://env.test/gql"); + }); + + it("prefers the explicit contentApiUrl option over the env var", async () => { + clearEnv(); + vi.stubEnv("SUPABASE_CONTENT_API_URL", "https://env.test/gql"); + const { config } = await supabaseMcpServer({ + contentApiUrl: "https://opt.test/gql", + }).createConfig({}); + const i = config.args.indexOf("--content-api-url"); + expect(config.args[i + 1]).toBe("https://opt.test/gql"); }); it("launches a local build dir with node when SUPABASE_MCP_SERVER_PATH is set", async () => { From 2c7b725d8504efa21eaa59a744e1115f5b027ec5 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 09:32:48 +0200 Subject: [PATCH 2/9] chore: stage the eval-source workspace glue under workspace/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verbatim copy of the eval-workspace repo's scripts/, patches/, and manifest (minus the two evals enabler patches: the local-build override shipped with the submodule PR, and content-api-url just landed as a real commit), plus the mise task file at the root and gitignore entries for the glue's artifacts. Re-rooting the scripts for life inside evals is the next commits' job — this one is the reviewable copy boundary. --- .gitignore | 6 + mise.toml | 102 +++++++ workspace/manifest.json | 28 ++ workspace/patches/README.md | 56 ++++ workspace/patches/mcp-content-api-url.patch | 40 +++ .../supabase-content-local-ports.patch | 46 +++ .../supabase-docs-guide-checksum.patch | 44 +++ .../supabase-docs-index-fail-closed.patch | 96 +++++++ .../supabase-docs-lint-warnings-skip.patch | 15 + .../supabase-docs-reference-dup-sources.patch | 135 +++++++++ workspace/scripts/ab-demo.sh | 73 +++++ workspace/scripts/ab-ready.sh | 51 ++++ workspace/scripts/ab-task.sh | 11 + workspace/scripts/ab.sh | 161 +++++++++++ workspace/scripts/ab.test.sh | 112 ++++++++ workspace/scripts/affected-task.sh | 8 + workspace/scripts/affected.ts | 199 +++++++++++++ workspace/scripts/apply-patches.sh | 145 ++++++++++ workspace/scripts/clone-docs.sh | 28 ++ workspace/scripts/docs-api.sh | 20 ++ workspace/scripts/docs-content-api.ts | 36 +++ workspace/scripts/docs-down.sh | 5 + workspace/scripts/docs-index.sh | 25 ++ workspace/scripts/docs-seed.sh | 35 +++ workspace/scripts/docs-up.sh | 13 + workspace/scripts/eval.sh | 10 + workspace/scripts/hooks.test.sh | 44 +++ workspace/scripts/load-keys.sh | 18 ++ workspace/scripts/manifest.mjs | 111 ++++++++ workspace/scripts/mcp-eval.sh | 10 + workspace/scripts/openai-preflight.sh | 36 +++ workspace/scripts/patches-lib.sh | 55 ++++ workspace/scripts/provenance.mjs | 266 ++++++++++++++++++ workspace/scripts/publish.sh | 138 +++++++++ workspace/scripts/setup.sh | 58 ++++ workspace/scripts/status.sh | 100 +++++++ workspace/scripts/status.test.sh | 59 ++++ workspace/scripts/store-key.sh | 29 ++ workspace/scripts/update.sh | 91 ++++++ 39 files changed, 2515 insertions(+) create mode 100644 mise.toml create mode 100644 workspace/manifest.json create mode 100644 workspace/patches/README.md create mode 100644 workspace/patches/mcp-content-api-url.patch create mode 100644 workspace/patches/supabase-content-local-ports.patch create mode 100644 workspace/patches/supabase-docs-guide-checksum.patch create mode 100644 workspace/patches/supabase-docs-index-fail-closed.patch create mode 100644 workspace/patches/supabase-docs-lint-warnings-skip.patch create mode 100644 workspace/patches/supabase-docs-reference-dup-sources.patch create mode 100755 workspace/scripts/ab-demo.sh create mode 100755 workspace/scripts/ab-ready.sh create mode 100755 workspace/scripts/ab-task.sh create mode 100755 workspace/scripts/ab.sh create mode 100755 workspace/scripts/ab.test.sh create mode 100755 workspace/scripts/affected-task.sh create mode 100755 workspace/scripts/affected.ts create mode 100755 workspace/scripts/apply-patches.sh create mode 100755 workspace/scripts/clone-docs.sh create mode 100755 workspace/scripts/docs-api.sh create mode 100644 workspace/scripts/docs-content-api.ts create mode 100755 workspace/scripts/docs-down.sh create mode 100755 workspace/scripts/docs-index.sh create mode 100755 workspace/scripts/docs-seed.sh create mode 100755 workspace/scripts/docs-up.sh create mode 100755 workspace/scripts/eval.sh create mode 100755 workspace/scripts/hooks.test.sh create mode 100755 workspace/scripts/load-keys.sh create mode 100755 workspace/scripts/manifest.mjs create mode 100755 workspace/scripts/mcp-eval.sh create mode 100755 workspace/scripts/openai-preflight.sh create mode 100644 workspace/scripts/patches-lib.sh create mode 100755 workspace/scripts/provenance.mjs create mode 100755 workspace/scripts/publish.sh create mode 100755 workspace/scripts/setup.sh create mode 100755 workspace/scripts/status.sh create mode 100755 workspace/scripts/status.test.sh create mode 100755 workspace/scripts/store-key.sh create mode 100755 workspace/scripts/update.sh diff --git a/.gitignore b/.gitignore index 932a6aea..40a83456 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,9 @@ dist/ results/*/ .sync-tmp/ + +# eval-source workspace glue (workspace/README.md) +/supabase/ +/results-ab/ +/.docs-index-stamp.json +/.publish/ diff --git a/mise.toml b/mise.toml new file mode 100644 index 00000000..137c5666 --- /dev/null +++ b/mise.toml @@ -0,0 +1,102 @@ +# supabase/evals — mise tasks (Supabase uses mise throughout; never a justfile). +# The eval-source workspace glue lives in workspace/ (see workspace/README.md): +# it wires docs, skills, and MCP-server sources into this harness for +# edit -> eval loops. Tasks run from the repo root. Flag-style args need `--`: +# mise run eval -- --eval --experiment + +[tools] +node = "22" +pnpm = "10" + +[tasks.setup] +description = "Install deps, init submodules (agent-skills + mcp), apply patches, wire .env, print status" +run = "workspace/scripts/setup.sh" + +[tasks.status] +description = "Host repo + submodule + clone state, env keys present, tooling" +run = "workspace/scripts/status.sh" + +[tasks.eval] +description = "Run evals with the workspace env applied (args pass to `pnpm eval`)" +run = "workspace/scripts/eval.sh" + +[tasks.update] +description = "Fetch + fast-forward the supabase clone, re-applying patches (-- --check = report only)" +run = "workspace/scripts/update.sh" + +[tasks.publish] +description = "Clean PR branch from patched-repo work: publish [--with ] (or --list)" +run = "workspace/scripts/publish.sh" + +[tasks.hooks-test] +description = "Self-test of the pre-push guard lifecycle (install/chain/reinstall; fixtures only, self-cleaning)" +run = "workspace/scripts/hooks.test.sh" + +[tasks.affected] +description = "Map changed skill/docs/mcp paths to a ready-to-run eval command" +run = "workspace/scripts/affected-task.sh" + +[tasks.ab] +description = "Head-to-head: [experiment] — edit applied vs reverted. No args = readiness probe" +run = "workspace/scripts/ab-task.sh" + +[tasks.ab-demo] +description = "Guided LIVE demo of the docs A/B loop (plants a canary doc, runs the real A/B, cleans up; asks before spending)" +run = "workspace/scripts/ab-demo.sh" + +[tasks.ab-test] +description = "Zero-cost self-test of the A/B runner (fakes the eval; only touches a clean skill file)" +run = "workspace/scripts/ab.test.sh" + +[tasks.status-test] +description = "Self-test of status.sh's Next/Ready diagnosis against synthetic workspace states" +run = "workspace/scripts/status.test.sh" + +[tasks.store-key] +description = "Store an API key in the keychain: store-key " +run = "workspace/scripts/store-key.sh" + +[tasks.apply-patches] +description = "(Re)apply the tracked enabler patches into the patched repos; idempotent" +run = "workspace/scripts/apply-patches.sh" + +# --- MCP-server loop (optional; not part of setup) --- + +[tasks.mcp-build] +description = "Build the mcp submodule with the enabler patches applied" +run = "git submodule update --init submodules/mcp && workspace/scripts/apply-patches.sh && pnpm mcp:build" + +[tasks.mcp-eval] +description = "Run evals against the local mcp build (auto init+patch+build)" +depends = ["mcp-build"] +run = "workspace/scripts/mcp-eval.sh" + +# --- Docs loop (optional, heavy — needs Docker + the supabase CLI) --- + +[tasks.clone-docs] +description = "Sparse-clone supabase/supabase (apps/docs + deps) and install. Run once" +run = "workspace/scripts/clone-docs.sh" + +[tasks.docs-up] +description = "Start the local content DB (ports 55321+) with docs migrations" +depends = ["clone-docs"] +run = "workspace/scripts/apply-patches.sh && workspace/scripts/docs-up.sh" + +[tasks.docs-down] +description = "Stop the local content DB" +run = "workspace/scripts/docs-down.sh" + +[tasks.docs-api] +description = "Serve the docs content GraphQL API at :3001 for search_docs" +depends = ["docs-up"] +run = "workspace/scripts/docs-api.sh" + +[tasks.docs-seed] +description = "Full docs embed — spends OpenAI credits; asks to confirm. Run once" +depends = ["docs-up"] +run = "workspace/scripts/docs-seed.sh" + +[tasks.docs-index] +description = "Incremental re-embed of changed docs pages (checksum-based, fail-closed)" +depends = ["docs-up"] +run = "workspace/scripts/docs-index.sh" diff --git a/workspace/manifest.json b/workspace/manifest.json new file mode 100644 index 00000000..38858b1d --- /dev/null +++ b/workspace/manifest.json @@ -0,0 +1,28 @@ +{ + "repos": { + "mcp": { + "dir": "submodules/mcp", + "kind": "submodule", + "patches": ["mcp-content-api-url"] + }, + "supabase": { + "dir": "supabase", + "remote": "git@github.com:supabase/supabase.git", + "patches": [ + "supabase-content-local-ports", + "supabase-docs-index-fail-closed", + "supabase-docs-guide-checksum", + "supabase-docs-lint-warnings-skip", + "supabase-docs-reference-dup-sources" + ], + "localPatches": [ + "supabase-content-local-ports", + "supabase-docs-lint-warnings-skip" + ] + }, + "skills": { + "dir": "submodules/agent-skills", + "kind": "submodule" + } + } +} diff --git a/workspace/patches/README.md b/workspace/patches/README.md new file mode 100644 index 00000000..88604275 --- /dev/null +++ b/workspace/patches/README.md @@ -0,0 +1,56 @@ +# Enabler patches + +Local changes to the cloned repos, materialized by `scripts/apply-patches.sh` +as identifiable **local commits** at the bottom of each clone's branch: + + [eval-workspace-local] dev shim — must never leave this machine + [eval-workspace-upstream] upstream candidate — leaves ONLY via + `mise run publish --with ` + +Your own work sits **above** these as normal commits, so `git commit -am` can +never sweep plumbing into it, and `mise run ab` can A/B any file (plumbing is +not in the working diff). A pre-push guard in every clone (and the agent-skills +submodule) blocks marker commits from being pushed; a pre-existing pre-push +hook is chained after the guard. Deliberate override (skips only the marker +checks): `EVAL_WORKSPACE_ALLOW_PUSH=1 git push …`. + +The `.patch` files here are **canonical**: `apply-patches` builds each commit +from the patch via the index, verifies the staged diff equals the patch before +committing (a user edit in a patch-owned file can never be absorbed), and on +every later run verifies the existing marker commit still matches the file. +`manifest.json` is the single source of truth mapping patches → repos / kinds +(read via `scripts/manifest.mjs`; marker subjects derive from the kind). + +| Patch | Repo | Files | Kind | Tests | What | +|---|---|---|---|---|---| +| `mcp-content-api-url` | supabase/mcp | `transports/stdio.ts` | upstream | 3 stdio integration tests (in the PR) | `--content-api-url` flag + `SUPABASE_CONTENT_API_URL` — **PR open: [mcp#343](https://github.com/supabase/mcp/pull/343)** | +| `evals-mcp-local-build-override` | supabase/evals | `core/index.ts`, `core/mcp-server.test.ts` | upstream | 4 `createConfig` tests | `SUPABASE_MCP_SERVER_PATH` local-build override (independent of the mcp flag; ships with the M1 submodule PR) | +| `evals-mcp-content-api-url` | supabase/evals | `core/index.ts`, `core/mcp-server.test.ts` | upstream | 2 `createConfig` tests | `contentApiUrl` option + `SUPABASE_CONTENT_API_URL` threading (applies on top of the override patch; upstream only after mcp's `--content-api-url` flag lands) | +| `supabase-content-local-ports` | supabase/supabase | `config.toml` | **LOCAL-ONLY** | n/a | dev ports 55321+ (avoid evals' 54321-9) | +| `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report | +| `supabase-docs-guide-checksum` | supabase/supabase | `guideModelLoader.ts` | upstream | none | guide checksum + `tryCatch` `onError` fix (docs-index edit detection) | +| `supabase-docs-lint-warnings-skip` | supabase/supabase | `lint-warnings-guide.ts` | **LOCAL-ONLY** | n/a | `DOCS_EMBED_ALLOW_MISSING_SOURCES` skip | +| `supabase-docs-reference-dup-sources` | supabase/supabase | `sources/reference-doc.ts` + test | upstream | **unit + real-output tests** | dedupe duplicate reference source paths (the 18-page `inserted 2/1` fix) | + +## Changing or regenerating a patch + +The marker commit and the `.patch` file must stay in sync — `apply-patches` +verifies both directions and fails loudly on drift. + +- **You changed the plumbing in the clone** (amended/edited the marker commit): + refresh the canonical file from the commit — + `git -C diff ^ > patches/.patch` + (plain `git diff` cannot see committed changes). +- **You edited the `.patch` file**: drop the old marker commit + (`git -C rebase --onto ^ `), then + `mise run apply-patches` re-creates it from the file. +- **A patch adds brand-new files**: when generating, `git add -N ` first, + then `git reset -- ` after — a leftover intent-to-add entry breaks `git stash`. + +## Upstreaming order + +`evals-mcp-local-build-override` — independent of everything, shipped with the +evals M1 submodule PR (evals#109). Then `mcp-content-api-url` PR → +`evals-mcp-content-api-url` PR (depends on the flag) → the three upstream +supabase fixes (independent; `guide-checksum` and `index-fail-closed` still +need regression tests; `reference-dup-sources` is test-covered and PR-ready). diff --git a/workspace/patches/mcp-content-api-url.patch b/workspace/patches/mcp-content-api-url.patch new file mode 100644 index 00000000..f29c6bc7 --- /dev/null +++ b/workspace/patches/mcp-content-api-url.patch @@ -0,0 +1,40 @@ +diff --git a/packages/mcp-server-supabase/src/transports/stdio.ts b/packages/mcp-server-supabase/src/transports/stdio.ts +index e974296..991c46f 100644 +--- a/packages/mcp-server-supabase/src/transports/stdio.ts ++++ b/packages/mcp-server-supabase/src/transports/stdio.ts +@@ -16,6 +16,7 @@ async function main() { + ['project-ref']: projectId, + ['read-only']: readOnly, + ['api-url']: apiUrl, ++ ['content-api-url']: cliContentApiUrl, + ['version']: showVersion, + ['features']: cliFeatures, + }, +@@ -34,6 +35,9 @@ async function main() { + ['api-url']: { + type: 'string', + }, ++ ['content-api-url']: { ++ type: 'string', ++ }, + ['version']: { + type: 'boolean', + }, +@@ -59,6 +63,9 @@ async function main() { + + const features = cliFeatures ? parseList(cliFeatures) : undefined; + ++ const contentApiUrl = ++ cliContentApiUrl ?? process.env.SUPABASE_CONTENT_API_URL; ++ + const platform = createSupabaseApiPlatform({ + accessToken, + apiUrl, +@@ -69,6 +76,7 @@ async function main() { + projectId, + readOnly, + features, ++ contentApiUrl, + }); + + const transport = new StdioServerTransport(); diff --git a/workspace/patches/supabase-content-local-ports.patch b/workspace/patches/supabase-content-local-ports.patch new file mode 100644 index 00000000..d6ee5bc4 --- /dev/null +++ b/workspace/patches/supabase-content-local-ports.patch @@ -0,0 +1,46 @@ +diff --git a/supabase/config.toml b/supabase/config.toml +index b03bf67c0d..e8f75e262b 100644 +--- a/supabase/config.toml ++++ b/supabase/config.toml +@@ -1,13 +1,13 @@ + # A string used to distinguish different Supabase projects on the same host. Defaults to the working + # directory name when running `supabase init`. +-project_id = "supabase" ++project_id = "eval-workspace-content" + + [remotes.prod] + project_id = "xguihxuzqibwxjnimxev" + + [api] + # Port to use for the API URL. +-port = 54321 ++port = 55321 + # Schemas to expose in your API. Tables, views and functions in this schema will get API + # endpoints. public and storage are always included. + schemas = ["public", "content", "storage", "graphql_public"] +@@ -22,14 +22,14 @@ schemas = ["public", "content", "storage", "graphql_public"] + + [db] + # Port to use for the local database URL. +-port = 54322 ++port = 55322 + # The database major version to use. This has to be the same as your remote database's. Run `SHOW + # server_version;` on the remote database to check. + major_version = 15 + + [studio] + # Port to use for Supabase Studio. +-port = 54323 ++port = 55323 + + openai_api_key = "env(OPENAI_API_KEY)" + +@@ -37,7 +37,7 @@ openai_api_key = "env(OPENAI_API_KEY)" + # are monitored, and you can view the emails that would have been sent from the web interface. + [inbucket] + # Port to use for the email testing server web interface. +-port = 54324 ++port = 55324 + + [storage] + # The maximum file size allowed (e.g. "5MB", "500KB"). diff --git a/workspace/patches/supabase-docs-guide-checksum.patch b/workspace/patches/supabase-docs-guide-checksum.patch new file mode 100644 index 00000000..c85530f6 --- /dev/null +++ b/workspace/patches/supabase-docs-guide-checksum.patch @@ -0,0 +1,44 @@ +diff --git a/apps/docs/resources/guide/guideModelLoader.ts b/apps/docs/resources/guide/guideModelLoader.ts +index 551bd98abf..219b2450a1 100644 +--- a/apps/docs/resources/guide/guideModelLoader.ts ++++ b/apps/docs/resources/guide/guideModelLoader.ts +@@ -1,6 +1,7 @@ + import matter from 'gray-matter' + import { promises as fs } from 'node:fs' + import { join, relative, resolve } from 'node:path' ++import { createHash } from 'node:crypto' + + import { extractMessageFromAnyError, FileNotFoundError, MultiError } from '~/app/api/utils' + import { preprocessMdxWithDefaults } from '~/features/directives/utils' +@@ -117,6 +118,11 @@ export class GuideModelLoader { + // Process MDX to get chunked sections for embedding + const { sections } = await processMdx(processedContent) + ++ // Content checksum so docs-index can detect guide edits (without this, ++ // GuideModel had no checksum -> normalized to '' -> every guide compared ++ // equal forever and incremental re-embed never fired for guides). ++ const checksum = createHash('sha256').update(processedContent).digest('base64') ++ + // Create subsections from the chunked sections + const subsections = sections.map((section) => ({ + title: section.heading, +@@ -133,6 +139,7 @@ export class GuideModelLoader { + return new GuideModel({ + title, + href, ++ checksum, + content: processedContent, + metadata, + subsections, +@@ -140,9 +147,9 @@ export class GuideModelLoader { + }, + (error) => { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { +- throw new FileNotFoundError('', error) ++ return new FileNotFoundError('', error) + } +- throw new Error( ++ return new Error( + `Failed to load guide from ${relPath}: ${extractMessageFromAnyError(error)}`, + { + cause: error, diff --git a/workspace/patches/supabase-docs-index-fail-closed.patch b/workspace/patches/supabase-docs-index-fail-closed.patch new file mode 100644 index 00000000..a1322c0e --- /dev/null +++ b/workspace/patches/supabase-docs-index-fail-closed.patch @@ -0,0 +1,96 @@ +diff --git a/apps/docs/scripts/search/generate-embeddings.ts b/apps/docs/scripts/search/generate-embeddings.ts +index d7519d2b15..4dfe6e6ba6 100644 +--- a/apps/docs/scripts/search/generate-embeddings.ts ++++ b/apps/docs/scripts/search/generate-embeddings.ts +@@ -118,6 +118,7 @@ function initSupabase(): SupabaseClient { + type PreparedSections = { + allSectionsToProcess: PageSectionForEmbedding[] + pageInfoMap: Map ++ preparationFailureCount: number + } + + async function prepareSections( +@@ -135,6 +136,7 @@ async function prepareSections( + + const allSectionsToProcess: PageSectionForEmbedding[] = [] + const pageInfoMap = new Map() ++ let preparationFailureCount = 0 + + for (const sourceBatch of createBatches(embeddingSources, CONFIG.SOURCE_CONCURRENCY)) { + await Promise.all( +@@ -238,6 +240,7 @@ async function prepareSections( + } catch (err) { + console.error(`Error preparing path '${path}' for processing.`) + console.error(err) ++ preparationFailureCount++ + } + }) + ) +@@ -246,7 +249,7 @@ async function prepareSections( + console.log( + `Prepared ${allSectionsToProcess.length} sections for processing from ${pageInfoMap.size} pages` + ) +- return { allSectionsToProcess, pageInfoMap } ++ return { allSectionsToProcess, pageInfoMap, preparationFailureCount } + } + + async function processAndInsertEmbeddings( +@@ -321,6 +324,9 @@ type BatchEmbeddingResult = { + processedCount: number + } + ++// Running total of embedding tokens billed across all batches, for a cost report. ++let embeddingTokenTotal = 0 ++ + async function processEmbeddingBatch( + openai: OpenAI, + batch: PageSectionForEmbedding[], +@@ -376,6 +382,7 @@ async function processEmbeddingBatch( + // Replace inputs with truncated inputs for downstream bookkeeping + for (let i = 0; i < inputs.length; i++) inputs[i] = truncatedInputs[i] + } ++ embeddingTokenTotal += embeddingResponse.usage?.total_tokens ?? 0 + + const { sectionsWithEmbeddings, failedSectionIndexes } = mapEmbeddingsToSections( + batch, +@@ -467,7 +474,7 @@ async function generateEmbeddings() { + : 'Checking which pages are new or have changed' + ) + +- const { allSectionsToProcess, pageInfoMap } = await prepareSections( ++ const { allSectionsToProcess, pageInfoMap, preparationFailureCount } = await prepareSections( + supabaseClient, + pageTable, + pageSectionTable, +@@ -493,10 +500,13 @@ async function generateEmbeddings() { + console.log( + `Page summary: ${processingResult.successfulPages.size} successful, ${processingResult.failedPages.size} failed` + ) ++ console.log( ++ `Embedding tokens billed: ${embeddingTokenTotal} (~$${((embeddingTokenTotal / 1_000_000) * 0.1).toFixed(4)} at $0.10/M for ${CONFIG.EMBEDDING_MODEL})` ++ ) + } catch (error) { + console.error('Critical error during embedding processing:', error) + console.log('Exiting due to complete processing failure') +- return ++ throw error + } + + console.log(`\nUpdating checksums for ${processingResult.successfulPages.size} successful pages`) +@@ -512,6 +522,16 @@ async function generateEmbeddings() { + + logFailedPages(pageInfoMap, processingResult) + ++ if ( ++ preparationFailureCount > 0 || ++ processingResult.failedPages.size > 0 || ++ successfulChecksumUpdates !== processingResult.successfulPages.size ++ ) { ++ throw new Error( ++ `Index incomplete: ${preparationFailureCount} source preparation failure(s), ${processingResult.failedPages.size} page failure(s), and ${processingResult.successfulPages.size - successfulChecksumUpdates} checksum update failure(s); refusing to purge old pages` ++ ) ++ } ++ + await purgeOldPages(supabaseClient, pageTable, refreshVersion) + + console.log('Embedding generation complete') diff --git a/workspace/patches/supabase-docs-lint-warnings-skip.patch b/workspace/patches/supabase-docs-lint-warnings-skip.patch new file mode 100644 index 00000000..70a3bf71 --- /dev/null +++ b/workspace/patches/supabase-docs-lint-warnings-skip.patch @@ -0,0 +1,15 @@ +diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts +index d28a57a27f..0207f87504 100644 +--- a/apps/docs/scripts/search/sources/lint-warnings-guide.ts ++++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts +@@ -29,6 +29,10 @@ export class LintWarningsGuideLoader extends BaseLoader { + + async load() { + if (!appId || !installationId || !privateKey) { ++ if (process.env.DOCS_EMBED_ALLOW_MISSING_SOURCES) { ++ console.warn('Skipping lint-warnings guide source: DOCS_GITHUB_APP_* not set (DOCS_EMBED_ALLOW_MISSING_SOURCES)') ++ return [] ++ } + throw new Error('Missing DOCS_GITHUB_APP_* environment variables') + } + diff --git a/workspace/patches/supabase-docs-reference-dup-sources.patch b/workspace/patches/supabase-docs-reference-dup-sources.patch new file mode 100644 index 00000000..b17ca5b3 --- /dev/null +++ b/workspace/patches/supabase-docs-reference-dup-sources.patch @@ -0,0 +1,135 @@ +diff --git a/apps/docs/scripts/search/sources/reference-doc.test.ts b/apps/docs/scripts/search/sources/reference-doc.test.ts +new file mode 100644 +index 0000000000..38f5cc9c0b +--- /dev/null ++++ b/apps/docs/scripts/search/sources/reference-doc.test.ts +@@ -0,0 +1,100 @@ ++import { mkdtemp, rm, writeFile } from 'node:fs/promises' ++import { existsSync } from 'node:fs' ++import { tmpdir } from 'node:os' ++import { join } from 'node:path' ++import { afterAll, describe, expect, it } from 'vitest' ++ ++import { loadClientLibReferenceFromNewPipeline } from './reference-doc' ++ ++/** ++ * Regression tests for duplicate entries in the new reference pipeline's ++ * sections.json: some trees repeat an entry (a category node and its overview ++ * child share id + slug), which used to emit two sources for one page path. ++ * Both sections then inserted under one page while the page's expected count ++ * came from the last source only, so the embed script flagged the page failed ++ * on every run (inserted 2/1) and the index could never complete. ++ */ ++ ++const tmpDirs: string[] = [] ++afterAll(async () => { ++ await Promise.all(tmpDirs.map((dir) => rm(dir, { recursive: true, force: true }))) ++}) ++ ++async function fixture(sections: unknown, functions: unknown): Promise { ++ const dir = await mkdtemp(join(tmpdir(), 'reference-doc-test-')) ++ tmpDirs.push(dir) ++ await writeFile(join(dir, 'sections.json'), JSON.stringify(sections)) ++ await writeFile(join(dir, 'functions.json'), JSON.stringify(functions)) ++ await writeFile(join(dir, 'typeSpec.json'), JSON.stringify({ methods: {} })) ++ return dir ++} ++ ++describe('loadClientLibReferenceFromNewPipeline', () => { ++ it('emits one source when a category node and its child repeat id + slug', async () => { ++ const contentDir = await fixture( ++ [ ++ { ++ id: 'auth-admin', ++ slug: 'auth-admin', ++ title: 'Auth Admin', ++ type: 'function', ++ items: [{ id: 'auth-admin', slug: 'auth-admin', title: 'Overview', type: 'function' }], ++ }, ++ ], ++ [{ id: 'auth-admin', title: 'Auth Admin', description: 'admin methods' }] ++ ) ++ ++ const sources = await loadClientLibReferenceFromNewPipeline({ ++ source: 'js-lib', ++ path: '/reference/javascript', ++ meta: { title: 'JavaScript Reference' }, ++ contentDir, ++ }) ++ ++ expect(sources.map((s) => s.path)).toEqual(['/reference/javascript/auth-admin']) ++ }) ++ ++ it('keeps the first flattened section when different ids share a slug', async () => { ++ const contentDir = await fixture( ++ [ ++ { id: 'first-id', slug: 'dup-slug', title: 'First', type: 'function' }, ++ { id: 'second-id', slug: 'dup-slug', title: 'Second', type: 'function' }, ++ ], ++ [ ++ { id: 'first-id', title: 'First fn', description: 'a' }, ++ { id: 'second-id', title: 'Second fn', description: 'b' }, ++ ] ++ ) ++ ++ const sources = await loadClientLibReferenceFromNewPipeline({ ++ source: 'js-lib', ++ path: '/reference/javascript', ++ meta: { title: 'JavaScript Reference' }, ++ contentDir, ++ }) ++ ++ expect(sources).toHaveLength(1) ++ expect(sources[0].path).toBe('/reference/javascript/dup-slug') ++ // deterministic winner: the first flattened section (matches the renderer's ++ // getFlattenedSections(...).find(...) lookup, which also takes the first) ++ expect(sources[0].extractTitle()).toBe('First fn') ++ }) ++ ++ // The real generated outputs are what production embeds; assert the invariant ++ // the page upsert depends on (path uniqueness) holds for them. ++ it.each(['javascript', 'dart'])('emits unique page paths for %s/v2', async (lib) => { ++ const contentDir = `content/reference/${lib}/v2` ++ if (!existsSync(join(contentDir, 'sections.json'))) return // sparse checkout without generated content ++ ++ const sources = await loadClientLibReferenceFromNewPipeline({ ++ source: `${lib}-lib`, ++ path: `/reference/${lib}`, ++ meta: { title: `${lib} Reference` }, ++ contentDir, ++ }) ++ ++ const paths = sources.map((s) => s.path) ++ expect(new Set(paths).size).toBe(paths.length) ++ expect(paths.length).toBeGreaterThan(0) ++ }) ++}) +diff --git a/apps/docs/scripts/search/sources/reference-doc.ts b/apps/docs/scripts/search/sources/reference-doc.ts +index 2e433746fe..eb81c75bcf 100644 +--- a/apps/docs/scripts/search/sources/reference-doc.ts ++++ b/apps/docs/scripts/search/sources/reference-doc.ts +@@ -332,14 +332,23 @@ export async function loadClientLibReferenceFromNewPipeline({ + + const flattened = flattenSections(refSections) + ++ // sections.json repeats some entries (a category node and its overview child ++ // share id + slug), and both resolve to the same spec entry, so keeping the ++ // duplicate would create two sources for one page path: sections double-insert ++ // and the page's inserted/expected bookkeeping fails. Keep the first. ++ const seenPaths = new Set() ++ + return flattened + .map((refSection) => { + const specSection = enriched.find((e) => e.id === refSection.id) + if (!specSection) return undefined ++ const pagePath = `${path}/${refSection.slug}` ++ if (seenPaths.has(pagePath)) return undefined ++ seenPaths.add(pagePath) + const titleForMeta = specSection.title || refSection.title + return new ClientLibReferenceSource( + source, +- `${path}/${refSection.slug}`, ++ pagePath, + refSection, + specSection, + { ...meta, slug: specSection.id, methodName: titleForMeta } diff --git a/workspace/scripts/ab-demo.sh b/workspace/scripts/ab-demo.sh new file mode 100755 index 00000000..d61cdff9 --- /dev/null +++ b/workspace/scripts/ab-demo.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Guided, self-cleaning LIVE demonstration of the docs A/B loop — the fastest +# way to see (and trust) the whole mechanism: +# +# 1. plants an un-guessable fact in ONE local docs page: a fictional package +# `@supabase/pinniped` for the fictional "Nimbus" runtime (append-only edit) +# 2. installs a matching throwaway eval (demo/canary-eval/) that PASSES only +# if the agent names that package +# 3. runs the real head-to-head (scripts/ab.sh): treatment (fact embedded in +# the local index) vs baseline (fact removed + re-embedded) +# 4. expected result: baseline FAIL 0/1 -> treatment PASS 1/1 — proof that a +# local docs edit alone moves an eval through search_docs +# 5. cleans up completely: guide reverted, canary de-embedded, eval removed +# +# Cost: 2 model runs (claude-sonnet-5) + a few embedding cents. Asks first. +set -euo pipefail +cd "$(dirname "$0")/.." + +GUIDE=supabase/apps/docs/content/guides/auth/choosing-a-server-package.mdx +EVAL_ID=investigate-workspace-canary-nimbus-package +EVAL_DST=evals/evals/$EVAL_ID + +fail() { echo "not ready: $1" >&2; echo "run \`mise run ab\` (no args) for the readiness probe + fix commands" >&2; exit 1; } + +# --- keys: direct keychain reads (load-keys can be flaky in foreground shells) --- +for k in ANTHROPIC_API_KEY OPENAI_API_KEY; do + v="$(security find-generic-password -a "$USER" -s "eval-workspace:$k" -w 2>/dev/null || true)" + [ -n "$v" ] && export "$k=$v" +done + +# --- hard preflight (same gates the A/B itself needs) --- +[ -e evals/.git ] || fail "evals not cloned" +[ -e supabase/.git ] || fail "supabase not cloned" +[ -n "${ANTHROPIC_API_KEY:-}" ] || fail "ANTHROPIC_API_KEY missing" +[ -n "${OPENAI_API_KEY:-}" ] || fail "OPENAI_API_KEY missing" +docker exec supabase_db_eval-workspace-content true 2>/dev/null || fail "content DB not running" +pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) +[ "${pages:-0}" -gt 0 ] 2>/dev/null || fail "docs index not seeded" +curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql || fail "docs-api not serving on :3001" +git -C supabase diff --quiet -- "${GUIDE#supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}" +[ ! -e "$EVAL_DST" ] || fail "$EVAL_DST already exists — remove it first" + +echo "This runs a live A/B: 2 claude-sonnet-5 runs + a few embedding cents, ~3-5 min." +read -r -p "Type 'demo' to proceed: " confirmation +[ "$confirmation" = demo ] || { echo "cancelled."; exit 0; } + +# --- plant the canary (append-only; reverted by the cleanup trap) --- +cleanup() { + echo "== demo cleanup: reverting canary, de-embedding, removing throwaway eval ==" + git -C supabase checkout -- "${GUIDE#supabase/}" 2>/dev/null || true + rm -rf "$EVAL_DST" + scripts/docs-index.sh >/dev/null 2>&1 \ + || echo "WARNING: cleanup re-embed failed — run \`mise run docs-index\` to de-embed the canary" >&2 +} +trap cleanup EXIT + +cat >> "$GUIDE" <<'EOF' + +## Nimbus workers + +- You're on a **Supabase-managed cron/queue worker** (the internal "Nimbus" runtime) → use **`@supabase/pinniped`**, the dedicated helper for Nimbus workers. +EOF +mkdir -p "$EVAL_DST" +cp demo/canary-eval/PROMPT.md demo/canary-eval/EVAL.ts "$EVAL_DST/" + +echo "== canary planted in $GUIDE; running the head-to-head ==" +scripts/ab.sh "$EVAL_ID" claude-sonnet-5 "$GUIDE" + +echo +echo "What you just saw: the ONLY difference between the two runs was that one" +echo "local docs page contained the planted fact. treatment PASS + baseline FAIL" +echo "means the agent found it via search_docs against YOUR local index — the" +echo "edit->embed->eval loop works end to end. Results: results-ab/$EVAL_ID.*.json" diff --git a/workspace/scripts/ab-ready.sh b/workspace/scripts/ab-ready.sh new file mode 100755 index 00000000..e9742e44 --- /dev/null +++ b/workspace/scripts/ab-ready.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# A/B readiness probe: which loops can you head-to-head right now, what's +# missing, and the exact command to fix each gap. `mise run ab` with no args +# lands here. Read-only; never mutates anything. +set -euo pipefail +cd "$(dirname "$0")/.." + +NEXT="" +ok() { printf ' [ok] %s\n' "$1"; } +miss() { printf ' [--] %s\n' "$1"; NEXT="${NEXT} $2\n"; } +have_key() { security find-generic-password -a "$USER" -s "eval-workspace:$1" >/dev/null 2>&1 || { [ -f .env ] && grep -qE "^$1=.+" .env; }; } + +echo "A/B readiness — mise run ab [experiment=claude-sonnet-5]" + +echo +echo "skills loop (edit evals/submodules/agent-skills/skills/…):" +if [ -e evals/submodules/agent-skills/.git ]; then ok "evals + agent-skills cloned"; else miss "evals/agent-skills not cloned" "mise run setup"; fi +if have_key ANTHROPIC_API_KEY; then ok "ANTHROPIC_API_KEY present"; else miss "ANTHROPIC_API_KEY missing" "mise run store-key ANTHROPIC_API_KEY"; fi + +echo +echo "mcp loop (edit mcp/packages/…):" +if [ -e mcp/.git ] && [ -d mcp/packages/mcp-server-supabase/dist ]; then + ok "local mcp cloned + built" +else + miss "local mcp not cloned/built" "mise run mcp-build" +fi + +echo +echo "docs loop (edit supabase/apps/docs/content/… pages):" +if [ -e supabase/.git ]; then ok "supabase (apps/docs) cloned"; else miss "supabase not cloned" "mise run clone-docs"; fi +if docker exec supabase_db_eval-workspace-content true 2>/dev/null; then + ok "content DB running" + pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) + if [ "${pages:-0}" -gt 0 ] 2>/dev/null; then ok "index seeded ($pages pages)"; else miss "index empty" "mise run docs-seed # one-time full embed (OpenAI \$; asks to confirm)"; fi +else + miss "content DB not running" "mise run docs-up" +fi +if curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql; then ok "docs-api serving :3001"; else miss "docs-api not serving" "mise run docs-api # keep it running in a separate terminal"; fi +if have_key OPENAI_API_KEY; then ok "OPENAI_API_KEY present"; else miss "OPENAI_API_KEY missing" "mise run store-key OPENAI_API_KEY"; fi + +echo +if [ -n "$NEXT" ]; then + echo "Next (for the loop you want):" + printf '%b' "$NEXT" | awk '!seen[$0]++' +else + echo "All loops ready." +fi +echo +echo "Then: make ONE edit (tracked file, unstaged), pick an eval (ls evals/evals/), run:" +echo " mise run ab " +echo "Wiring self-test (free): mise run ab-test · guided live demo: mise run ab-demo" diff --git a/workspace/scripts/ab-task.sh b/workspace/scripts/ab-task.sh new file mode 100755 index 00000000..7252140a --- /dev/null +++ b/workspace/scripts/ab-task.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# mise task wrapper: ab [experiment=claude-sonnet-5]. +# No args -> the readiness probe (which loops are runnable + how to fix gaps). +# Reorders to scripts/ab.sh's . For multiple edited +# files or other advanced use, call scripts/ab.sh directly. +set -euo pipefail +cd "$(dirname "$0")/.." + +[ $# -gt 0 ] || exec scripts/ab-ready.sh +[ $# -ge 2 ] || { echo "usage: mise run ab [experiment] (no args = readiness probe)" >&2; exit 2; } +exec scripts/ab.sh "$1" "${3:-claude-sonnet-5}" "$2" diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh new file mode 100755 index 00000000..d951edc7 --- /dev/null +++ b/workspace/scripts/ab.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Head-to-head eval: baseline (your edit reverted) vs treatment (edit applied). +# Same eval, same experiment; the ONLY difference is your uncommitted edit. +# +# Usage: scripts/ab.sh [more-paths...] +# is a file inside a clone; its clone selects the loop + how to +# re-sync between states: +# supabase/apps/docs/content/… docs loop (re-embed via docs-index; needs `mise run docs-api` up) +# mcp/… mcp loop (rebuild the local server) +# evals/submodules/agent-skills/… skills loop (no re-sync; read live via symlink) +# +# The enabler patches live as [eval-workspace-*] commits below your work (see +# apply-patches.sh), so your unstaged edit is the ONLY working diff — stashing +# it for baseline can't disturb the plumbing, even on patch-owned files. +# +# Sync steps keep their normal exit semantics: if docs-index or the mcp build +# fails, the A/B aborts (and the restore trap puts your edit back). No special +# tolerance — a failing sync means the states can't be trusted. +# +# AB_DRYRUN=1 prints the resolved plan and exits (no runs, no side effects). +# AB_EVAL_CMD / AB_SYNC_CMD override the eval / sync step (testing; see ab.test.sh). +set -euo pipefail +cd "$(dirname "$0")/.." + +[ $# -ge 3 ] || { echo "usage: scripts/ab.sh [more-paths...]" >&2; exit 2; } +EVAL="$1"; EXP="$2"; shift 2 +PATHS=("$@") + +# --- keys: direct keychain reads (robust vs load-keys foreground flakiness) --- +for k in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do + v="$(security find-generic-password -a "$USER" -s "eval-workspace:$k" -w 2>/dev/null || true)" + [ -n "$v" ] && export "$k=$v" +done + +# --- clone + loop from the first path --- +case "${PATHS[0]}" in + supabase/apps/docs/content/*) LOOP=docs; CLONE=supabase; STRIP=supabase/; PREFIX=supabase/apps/docs/content/ ;; + supabase/apps/docs/*) echo "docs A/B works on content pages (supabase/apps/docs/content/…) — other docs files aren't part of the embed loop: ${PATHS[0]}" >&2; exit 2 ;; + mcp/*) LOOP=mcp; CLONE=mcp; STRIP=mcp/; PREFIX=mcp/ ;; + evals/submodules/agent-skills/*) LOOP=skills; CLONE=evals/submodules/agent-skills; STRIP=evals/submodules/agent-skills/; PREFIX=evals/submodules/agent-skills/ ;; + *) echo "path must be under supabase/apps/docs/content/, mcp/, or evals/submodules/agent-skills/: ${PATHS[0]}" >&2; exit 2 ;; +esac + +# every path must live in this loop's editable scope; build clone-relative paths +REL=() +for p in "${PATHS[@]}"; do + case "$p" in + "$PREFIX"*) REL+=("${p#$STRIP}") ;; + *) echo "all paths must be under $PREFIX (the loop chosen by ${PATHS[0]}): $p" >&2; exit 2 ;; + esac +done + +MCP="$PWD/mcp/packages/mcp-server-supabase" +CONTENT_URL="http://127.0.0.1:3001/docs/api/graphql" +RUN_ENV=() +case "$LOOP" in + docs) RUN_ENV=( "SUPABASE_MCP_SERVER_PATH=$MCP" "SUPABASE_CONTENT_API_URL=$CONTENT_URL" ) ;; + mcp) RUN_ENV=( "SUPABASE_MCP_SERVER_PATH=$MCP" ) ;; +esac + +sync() { + if [ -n "${AB_SYNC_CMD:-}" ]; then bash -c "$AB_SYNC_CMD"; return; fi # override/test hook + case "$LOOP" in + docs) scripts/docs-index.sh ;; + mcp) ( cd mcp && pnpm build ) ;; + skills) : ;; + esac +} + +if [ -n "${AB_DRYRUN:-}" ]; then + echo "eval=$EVAL experiment=$EXP loop=$LOOP clone=$CLONE" + echo "revert paths: ${REL[*]}" + echo "run env: ${RUN_ENV[*]:-（none）}" + exit 0 +fi + +: "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY not in keychain — see README}" +# A/B reverts tracked, unstaged working-tree edits only, via a scoped git stash. +# `stash push -- ` merges whole-index state, so ANY staged entry in the +# clone (even an unrelated intent-to-add) breaks it. Require a clean index. +git -C "$CLONE" diff --cached --quiet --ita-visible-in-index \ + || { echo "the $CLONE index has staged changes (see: git -C $CLONE status) — unstage them first; ab's scoped stash needs a clean index" >&2; exit 1; } +for r in "${REL[@]}"; do + git -C "$CLONE" ls-files --error-unmatch -- "$r" >/dev/null 2>&1 \ + || { echo "not a tracked file in $CLONE: $r (ab reverts tracked edits; commit a brand-new file first, or revert it by hand)" >&2; exit 1; } + if git -C "$CLONE" diff --quiet -- "$r"; then + echo "no unstaged edit at $r (nothing to A/B)" >&2; exit 1 + fi +done + + +if [ "$LOOP" = docs ]; then + : "${OPENAI_API_KEY:?OPENAI_API_KEY not in keychain — the docs re-embed needs it}" + curl -sf -o /dev/null "$CONTENT_URL" || { echo "docs-api not reachable on :3001 — run \`mise run docs-api\` in another terminal first" >&2; exit 1; } + [ -d "$MCP/dist" ] || { echo "building local mcp (needed for search_docs routing)…"; ( cd mcp && pnpm install && pnpm build ); } +fi + +RES="evals/results/$EXP/$EVAL.json" +OUT="results-ab"; mkdir -p "$OUT" + +run_eval() { # $1 = label + sync + if [ -n "${AB_EVAL_CMD:-}" ]; then + RES="$RES" AB_LABEL="$1" bash -c "$AB_EVAL_CMD" # override/test hook; must write $RES + else + ( cd evals && env ${RUN_ENV[@]+"${RUN_ENV[@]}"} pnpm eval --eval "$EVAL" --experiment "$EXP" --runs 1 ) + fi + [ -f "$RES" ] || { echo "no result at $RES — check the eval/experiment ids" >&2; exit 1; } + cp "$RES" "$OUT/$EVAL.$1.json" + # per-arm receipt: capture provenance AFTER this arm's sync, so baseline and + # treatment records differ by exactly the edit under test (report-only) + node scripts/provenance.mjs --embed "$OUT/$EVAL.$1.json" +} + +# Restoration is ONE idempotent path: pop the stash AND re-sync, so the index/ +# build never stays at baseline while the tree shows treatment. The EXIT trap +# runs it on any post-stash failure, preserving the original exit status. +STASHED=0 +restore() { + [ "$STASHED" = 1 ] || return 0 + echo "== restoring edit ==" + if git -C "$CLONE" stash pop -q; then + STASHED=0 + else + echo "ERROR: stash pop failed — your edit is in: git -C $CLONE stash list" >&2 + return 1 + fi + if ! sync; then + echo "ERROR: post-restore re-sync failed — the index/build does not reflect your edit (docs: scripts/docs-index.sh; mcp: pnpm -C mcp build)" >&2 + return 1 + fi +} +# Preserve the run's own failure status; a clean run that fails cleanup exits nonzero. +trap 'rc=$?; if ! restore && [ "$rc" -eq 0 ]; then rc=1; fi; exit $rc' EXIT + +echo "== treatment: edit applied (${PATHS[*]}) ==" +run_eval treatment + +echo "== reverting edit for baseline ==" +git -C "$CLONE" stash push -q -- "${REL[@]}" +STASHED=1 + +echo "== baseline: edit reverted ==" +run_eval baseline + +restore # pop + re-sync now; the EXIT trap becomes a no-op + +EVAL="$EVAL" EXP="$EXP" OUT="$OUT" node -e ' +const path=require("path"); +const f=(p)=>{try{return require(path.resolve(p))}catch{return null}}; +const {EVAL,EXP,OUT}=process.env; +const b=f(`${OUT}/${EVAL}.baseline.json`), t=f(`${OUT}/${EVAL}.treatment.json`); +const chk=(r)=>{const c=(r&&r.checks)||[];return `${c.filter(x=>x&&x.passed).length}/${c.length}`}; +const row=(l,r)=>`${l.padEnd(10)} passed=${String(r&&r.passed).padEnd(5)} checks=${chk(r).padEnd(6)} docs.calls=${((r&&r.docs&&r.docs.calls)||[]).length}`; +console.log(`\n=== A/B: ${EVAL} (${EXP}) ===`); +console.log(row("baseline",b)); +console.log(row("treatment",t)); +const d=((t&&t.passed)?1:0)-((b&&b.passed)?1:0); +console.log(d>0?"\n-> edit IMPROVED the eval (FAIL->PASS)":d<0?"\n-> edit REGRESSED the eval (PASS->FAIL)":"\n-> no pass/fail change (compare checks / docs.calls above)"); +console.log(`saved: ${OUT}/${EVAL}.{baseline,treatment}.json`); +' diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh new file mode 100755 index 00000000..a9c40761 --- /dev/null +++ b/workspace/scripts/ab.test.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Zero-cost integration test for scripts/ab.sh. Fakes the eval via AB_EVAL_CMD +# (no model, no OpenAI, no services) and proves: the scoped stash/restore + result +# copy + comparison work, AND the selected edit and index are byte-identical after +# the run — including when a run fails mid-A/B (trap must restore). +# +# SAFETY: only ever touches a tracked skill file that is currently CLEAN (no +# working-tree or staged changes); it never checks out a file with your edits. +# If no clean skill file exists it SKIPS. Skills loop → sync is a no-op, so +# nothing else in the workspace is affected. Run: bash scripts/ab.test.sh +set -euo pipefail +cd "$(dirname "$0")/.." + +SK=evals/submodules/agent-skills +[ -e "$SK/.git" ] || { echo "SKIP: agent-skills not cloned (run: mise run setup)"; exit 0; } + +# pick a tracked SKILL.md with NO local changes, so restoring it can't lose work +REL="" +for c in $(git -C "$SK" ls-files 'skills/*/SKILL.md'); do + if git -C "$SK" diff --quiet -- "$c" && git -C "$SK" diff --cached --quiet -- "$c"; then REL="$c"; break; fi +done +[ -n "$REL" ] || { echo "SKIP: no CLEAN tracked skill file to test with"; exit 0; } +FILE="$SK/$REL" +EVAL=ab-selftest; EXP=claude-sonnet-5 +RES="evals/results/$EXP/$EVAL.json" + +# only now (after selecting a clean file) arm cleanup: revert our own edit, drop fixtures +MCPREL=packages/mcp-server-supabase/src/transports/stdio.ts +cleanup() { + git -C "$SK" checkout -q -- "$REL" 2>/dev/null || true + [ -e mcp/.git ] && git -C mcp checkout -q -- "$MCPREL" 2>/dev/null || true + rm -f "$RES" "results-ab/$EVAL".*.json +} +trap cleanup EXIT + +# fake eval: treatment passes, baseline fails; hard-fail baseline if AB_FAIL_BASELINE set. +FAKE='mkdir -p "$(dirname "$RES")"; if [ "$AB_LABEL" = baseline ] && [ -n "${AB_FAIL_BASELINE:-}" ]; then exit 7; fi; if [ "$AB_LABEL" = treatment ]; then p=true; else p=false; fi; printf "{\"passed\":%s,\"checks\":[{\"name\":\"x\",\"passed\":%s}],\"docs\":{\"calls\":[1,2]}}" "$p" "$p" > "$RES"' + +pass=0; fail=0 +ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $1 (want[$3] got[$2])"; fi; } + +# make an unstaged edit on the clean file +printf '\n\n' >> "$FILE" +edited=$(git -C "$SK" hash-object "$REL") +index0=$(git -C "$SK" diff --cached -- "$REL") # empty (was clean) + +# observable sync hook: every sync appends a line (treatment, baseline, restore = 3) +CNT=/tmp/ab_selftest_sync.cnt +SYNC="echo x >> $CNT" + +# --- happy path --- +: > "$CNT" +ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest.out 2>&1 \ + || { echo "FAIL: ab exited nonzero (happy path)"; sed 's/^/ /' /tmp/ab_selftest.out; fail=$((fail+1)); } +ck "edit preserved after success" "$(git -C "$SK" hash-object "$REL")" "$edited" +ck "index untouched after success" "$(git -C "$SK" diff --cached -- "$REL")" "$index0" +ck "treatment result copied" "$([ -f "results-ab/$EVAL.treatment.json" ] && echo y || echo n)" "y" +ck "baseline result copied" "$([ -f "results-ab/$EVAL.baseline.json" ] && echo y || echo n)" "y" +ck "reports IMPROVED (FAIL->PASS)" "$(grep -c 'IMPROVED' /tmp/ab_selftest.out)" "1" +ck "checks rendered (treatment 1/1)" "$(grep -c 'treatment.*checks=1/1' /tmp/ab_selftest.out)" "1" +ck "sync ran 3x (incl. post-restore)" "$(wc -l < "$CNT" | tr -d ' ')" "3" + +# --- failure path: baseline dies AFTER stash; ab's trap must restore the edit AND re-sync --- +: > "$CNT" +AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest2.out 2>&1 || true +ck "edit restored after mid-run failure" "$(git -C "$SK" hash-object "$REL")" "$edited" +ck "index untouched after failure" "$(git -C "$SK" diff --cached -- "$REL")" "$index0" +ck "trap re-synced after failure (3x)" "$(wc -l < "$CNT" | tr -d ' ')" "3" +ck "failure exit status preserved" "$(AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/dev/null 2>&1; echo $?)" "7" + +# --- restore-sync fails (3rd sync): must fail the run, keep the edit, skip the report --- +: > "$CNT" +SYNC_FAIL3="echo x >> $CNT; [ \$(wc -l < $CNT) -lt 3 ] || { echo sync3-boom >&2; exit 9; }" +rc3=$(ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC_FAIL3" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest3.out 2>&1; echo $?) +ck "restore-sync failure fails the run" "$([ "$rc3" -ne 0 ] && echo nonzero || echo zero)" "nonzero" +ck "edit intact after restore-sync failure" "$(git -C "$SK" hash-object "$REL")" "$edited" +ck "reports re-sync error" "$(grep -c 'post-restore re-sync failed' /tmp/ab_selftest3.out)" "1" +ck "no A/B report on failed restore" "$(grep -c '=== A/B' /tmp/ab_selftest3.out)" "0" + +# --- docs loop must reject a second path outside the content scope --- +rc4=$(AB_DRYRUN=1 bash scripts/ab.sh e x supabase/apps/docs/content/a.mdx supabase/apps/studio/foo.ts >/dev/null 2>&1; echo $?) +ck "rejects non-content path in docs loop" "$rc4" "2" + +# --- dirty clone index (e.g. intent-to-add residue) must be refused before stashing --- +echo probe > "$SK/.ab-selftest-idx-probe" +git -C "$SK" add -N .ab-selftest-idx-probe +rc5=$(ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest5.out 2>&1; echo $?) +git -C "$SK" reset -q -- .ab-selftest-idx-probe && rm -f "$SK/.ab-selftest-idx-probe" +ck "refuses dirty clone index" "$rc5" "1" +ck "explains staged-index refusal" "$(grep -c 'staged changes' /tmp/ab_selftest5.out)" "1" + +# --- patch-owned file A/B (the commit model's core win): an unstaged edit on a +# file the mcp enabler patch owns must A/B cleanly, and the plumbing marker +# commit must be untouched on both success and mid-run failure --- +if [ -e mcp/.git ] && git -C mcp diff --quiet -- "$MCPREL" && git -C mcp diff --cached --quiet --ita-visible-in-index; then + marker0=$(git -C mcp rev-parse HEAD) + printf '\n// ab-selftest edit\n' >> "mcp/$MCPREL" + edited2=$(git -C mcp hash-object "$MCPREL") + ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "mcp/$MCPREL" >/tmp/ab_selftest6.out 2>&1 \ + || { echo "FAIL: patch-owned A/B exited nonzero"; sed 's/^/ /' /tmp/ab_selftest6.out; fail=$((fail+1)); } + ck "patch-owned: edit preserved" "$(git -C mcp hash-object "$MCPREL")" "$edited2" + ck "patch-owned: marker commit intact" "$(git -C mcp rev-parse HEAD)" "$marker0" + AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "mcp/$MCPREL" >/dev/null 2>&1 || true + ck "patch-owned: edit restored after failure" "$(git -C mcp hash-object "$MCPREL")" "$edited2" + ck "patch-owned: marker intact after failure" "$(git -C mcp rev-parse HEAD)" "$marker0" + git -C mcp checkout -q -- "$MCPREL" +else + echo "SKIP: mcp not cloned/clean — patch-owned A/B regression not run" +fi + +echo "ab.test: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/workspace/scripts/affected-task.sh b/workspace/scripts/affected-task.sh new file mode 100755 index 00000000..02c0782f --- /dev/null +++ b/workspace/scripts/affected-task.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Which evals does a change affect? Maps changed skill/docs/mcp paths to a +# ready-to-run eval command. e.g. mise run affected apps/docs/content/guides/auth/x.mdx +set -euo pipefail +cd "$(dirname "$0")/.." +ROOT="$PWD" + +cd evals/apps/framework && exec node --import tsx/esm "$ROOT/scripts/affected.ts" "$@" diff --git a/workspace/scripts/affected.ts b/workspace/scripts/affected.ts new file mode 100755 index 00000000..7683e21d --- /dev/null +++ b/workspace/scripts/affected.ts @@ -0,0 +1,199 @@ +#!/usr/bin/env tsx +const { + existsSync, + readFileSync, + readdirSync, + statSync, +} = require("node:fs"); +const { createRequire } = require("node:module"); +const { basename, dirname, join } = require("node:path"); +const { pathToFileURL } = require("node:url"); + +const EVALS_ROOT = join(dirname(process.argv[1]), "..", "evals"); +const requireFromFramework = createRequire(join(EVALS_ROOT, "apps/framework/package.json")); +const ALIASES: Record = { + functions: "edge-functions", + rest: "data-api", + api: "data-api", +}; + +type Experiment = { name: string; skills: string[] }; +type Eval = { + id: string; + product: string[]; + topic: string[]; + interface?: string; +}; + +async function loadExperiments(): Promise { + const dir = join(EVALS_ROOT, "experiments"); + const experiments: Experiment[] = []; + + for (const file of readdirSync(dir).filter((file) => file.endsWith(".ts")).sort()) { + // Experiment files are runtime-selected plugins, matching run-eval.ts discovery. + const module = await import(pathToFileURL(join(dir, file)).href); + const config = module.default as { skills?: string[] }; + experiments.push({ + name: file.replace(/\.ts$/, ""), + skills: config.skills ?? [], + }); + } + + return experiments; +} + +async function discoverEvals(): Promise { + // Resolve from evals' workspace because this script intentionally lives outside that package. + const parserPath = requireFromFramework.resolve("@supabase-evals/core/eval-markdown"); + const { parseEvalMarkdown } = await import(pathToFileURL(parserPath).href); + const dir = join(EVALS_ROOT, "evals"); + if (!existsSync(dir)) return []; + + const evals: Eval[] = []; + for (const id of readdirSync(dir).sort()) { + const evalDir = join(dir, id); + if (!statSync(evalDir).isDirectory()) continue; + + const promptPath = join(evalDir, "PROMPT.md"); + const metadata = parseEvalMarkdown( + readFileSync(promptPath, "utf8"), + `evals/${id}/PROMPT.md`, + ).metadata; + evals.push({ + id, + product: metadata.product, + topic: metadata.topic, + interface: metadata.interface, + }); + } + + return evals; +} + +function addPathTokens( + segments: string[], + vocabulary: Set, + destination: Set, +): void { + for (const rawSegment of segments) { + const segment = rawSegment.replace(/\.[^.]+$/, ""); + const token = ALIASES[segment] ?? segment; + if (vocabulary.has(token)) destination.add(token); + } +} + +function matchingEvalIds(evals: Eval[], tokens: Set): string[] { + return evals + .filter((entry) => [...entry.product, ...entry.topic].some((token) => tokens.has(token))) + .map((entry) => entry.id); +} + +async function main(): Promise { + const paths = [...new Set(process.argv.slice(2))]; + if (paths.length === 0) { + console.log("Usage: affected.ts ..."); + return; + } + + const [experiments, evals] = await Promise.all([ + loadExperiments(), + discoverEvals(), + ]); + const vocabulary = new Set(evals.flatMap((entry) => [...entry.product, ...entry.topic])); + const skills = new Set(); + const docsTokens = new Set(); + const mcpTokens = new Set(); + const unknownPaths: string[] = []; + let allMcpEvals = false; + let serverWide = false; + + for (const rawPath of paths) { + const path = rawPath.replaceAll("\\", "/"); + let recognized = false; + + const skillMatch = path.match(/(?:^|\/)skills\/([^/]+)(?:\/|$)/); + if (skillMatch) { + skills.add(skillMatch[1]); + recognized = true; + } + + const docsMarker = "apps/docs/content/"; + const docsIndex = path.indexOf(docsMarker); + if (docsIndex !== -1) { + addPathTokens(path.slice(docsIndex + docsMarker.length).split("/"), vocabulary, docsTokens); + recognized = true; + } + + const isMcp = + path.includes("mcp-server-supabase/src/") || + path.includes("/tools/") || + path.startsWith("tools/") || + /^src\/(?:server\.ts|index\.ts|transports\/)/.test(path); + if (isMcp) { + recognized = true; + const file = basename(path); + const stem = file.replace(/\.ts$/, ""); + + if (file === "server.ts" || file === "index.ts" || /(?:^|\/)transports\//.test(path)) { + serverWide = true; + } else if (stem === "docs-tools") { + allMcpEvals = true; + } else { + addPathTokens(stem.split("-"), vocabulary, mcpTokens); + } + } + + if (!recognized) unknownPaths.push(rawPath); + } + + if (unknownPaths.length > 0) { + console.log(`Ignored unknown paths: ${unknownPaths.join(", ")}`); + } + + let commandCount = 0; + + if (skills.size > 0) { + const names = experiments + .filter((experiment) => experiment.skills.some((skill) => skills.has(skill))) + .map((experiment) => experiment.name); + if (names.length > 0) { + console.log(`mise run eval -- ${names.map((name) => `--experiment ${name}`).join(" ")} --suite regression`); + commandCount++; + } else { + console.log(`No experiments use changed skills: ${[...skills].sort().join(", ")}`); + } + } + + if (docsTokens.size > 0) { + const ids = matchingEvalIds(evals, docsTokens); + if (ids.length > 0) { + console.log(`mise run eval -- ${ids.map((id) => `--eval ${id}`).join(" ")}`); + commandCount++; + } + } + + if (mcpTokens.size > 0 || allMcpEvals) { + const ids = new Set(matchingEvalIds(evals, mcpTokens)); + if (allMcpEvals) { + for (const entry of evals) { + if (entry.interface === "mcp") ids.add(entry.id); + } + } + if (ids.size > 0) { + console.log(`mise run eval -- ${[...ids].sort().map((id) => `--eval ${id}`).join(" ")}`); + commandCount++; + } + } + + if (serverWide) { + console.log("mise run eval -- --smoke"); + commandCount++; + } + + if (commandCount === 0) console.log("No affected evals."); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/workspace/scripts/apply-patches.sh b/workspace/scripts/apply-patches.sh new file mode 100755 index 00000000..6c724acc --- /dev/null +++ b/workspace/scripts/apply-patches.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# Materialize the enabler patches as identifiable LOCAL COMMITS in each clone: +# [eval-workspace-local] dev shim — must never leave this machine +# [eval-workspace-upstream] upstream candidate — leaves ONLY via +# `mise run publish … --with ` (reworded) +# Your own work sits ABOVE these as normal commits, so `git commit -am` can +# never sweep plumbing into it, and stash/A-B isolation stays clean. +# +# Idempotent: a patch whose marker commit already exists is skipped; patch +# content found uncommitted in the working tree (the old model) is migrated +# into a commit. Also installs a pre-push guard in every publishable repo +# (incl. the agent-skills submodule) that blocks marker commits from being +# pushed anywhere. +set -euo pipefail +cd "$(dirname "$0")/.." +ROOT="$PWD" +source scripts/patches-lib.sh + +apply_one() { + local repo="$1" patch="$2" subject sha h s + subject=$(patch_subject "$patch") + # marker commit already present? (search ALL local-only history) + sha="" + while read -r h s; do + [ "$s" = "$subject" ] && { sha="$h"; break; } + done </dev/null) +EOF + if [ -n "$sha" ]; then + # the .patch file is canonical — the commit must still match it EXACTLY: + # rebuild parent-tree + patch in a temp index and compare tree OIDs + local tmpidx want got + tmpidx=$(mktemp /tmp/eval-workspace-idx-XXXXXX); rm -f "$tmpidx" # keep only the name: git must create the index itself + want=$(git -C "$repo" rev-parse "$sha^{tree}") + got="" + if GIT_INDEX_FILE="$tmpidx" git -C "$repo" read-tree "$sha^" 2>/dev/null \ + && GIT_INDEX_FILE="$tmpidx" git -C "$repo" apply --cached "$ROOT/$patch" 2>/dev/null; then + got=$(GIT_INDEX_FILE="$tmpidx" git -C "$repo" write-tree 2>/dev/null || echo "") + fi + rm -f "$tmpidx" + if [ "$got" = "$want" ]; then + echo " $(basename "$patch") already committed in $repo" + else + echo " ERROR: the $(basename "$patch") commit in $repo differs from the canonical patch file." >&2 + echo " refresh the file from the commit: git -C $repo diff $sha^ $sha > $patch" >&2 + echo " or drop the commit and re-apply: git -C $repo rebase --onto $sha^ $sha && mise run apply-patches" >&2 + exit 1 + fi + return + fi + # Build the commit from the CANONICAL patch via the index — never stage whole + # files, so user edits sitting in patch-owned files can't be absorbed. + if git -C "$repo" apply --index --check "$ROOT/$patch" 2>/dev/null; then + git -C "$repo" apply --index "$ROOT/$patch" # fresh clone: index + worktree + elif git -C "$repo" apply --cached --check "$ROOT/$patch" 2>/dev/null; then + git -C "$repo" apply --cached "$ROOT/$patch" # content already in worktree; extra edits stay unstaged above + else + echo " ERROR: $(basename "$patch") does not apply cleanly to $repo — regenerate it (patches/README.md)" >&2 + exit 1 + fi + # the staged diff must be exactly the canonical patch (payload comparison) + if ! diff -q <(git -C "$repo" diff --cached | grep -E '^[+-]' | grep -vE '^(\+\+\+|---)') \ + <(grep -E '^[+-]' "$ROOT/$patch" | grep -vE '^(\+\+\+|---)') >/dev/null 2>&1; then + git -C "$repo" reset -q + echo " ERROR: staged diff for $(basename "$patch") differs from the canonical patch — aborted (index reset)" >&2 + exit 1 + fi + GIT_AUTHOR_NAME=eval-workspace GIT_AUTHOR_EMAIL=eval-workspace@local \ + GIT_COMMITTER_NAME=eval-workspace GIT_COMMITTER_EMAIL=eval-workspace@local \ + git -C "$repo" commit -q -m "$subject" \ + -m "eval-workspace plumbing ($(patch_kind "$patch") kind), generated from $patch. Do not push; see patches/README.md." + echo " committed $(basename "$patch") -> $repo" +} + +install_pre_push_guard() { + local dir="$1" hooks hook + hooks=$(git -C "$dir" rev-parse --path-format=absolute --git-path hooks) # submodule .git is a file + mkdir -p "$hooks" + hook="$hooks/pre-push" + if [ -e "$hook" ] && ! grep -q 'eval-workspace pre-push guard' "$hook"; then + if [ -e "$hooks/pre-push.eval-workspace-chained" ] && ! cmp -s "$hook" "$hooks/pre-push.eval-workspace-chained"; then + echo " ERROR: $dir has BOTH a foreign pre-push hook AND a different previously-chained one — merge them manually:" >&2 + echo " foreign: $hook" >&2 + echo " chained: $hooks/pre-push.eval-workspace-chained" >&2 + exit 1 + fi + mv "$hook" "$hooks/pre-push.eval-workspace-chained" + echo " note: existing pre-push hook in $dir now chained after the eval-workspace guard" + fi + cat > "$hook" <<'HOOK' +#!/usr/bin/env bash +# eval-workspace pre-push guard (generated by scripts/apply-patches.sh). +# Blocks eval-workspace plumbing commits from leaving this machine. Publish work +# upstream via a clean cherry-picked branch: +# mise run publish [--with ] +# Deliberate override (skips ONLY the marker checks, never a chained hook): +# EVAL_WORKSPACE_ALLOW_PUSH=1 git push … +input=$(cat) +zero=0000000000000000000000000000000000000000 +if [ -z "${EVAL_WORKSPACE_ALLOW_PUSH:-}" ]; then + while read -r _lref lsha _rref _rsha; do + [ -z "$lsha" ] && continue + [ "$lsha" = "$zero" ] && continue # deleting a remote ref + # scan the FULL ancestry being pushed: markers only ever exist locally, so + # any marker anywhere below $lsha means plumbing would leave the machine. + # A failing log is a block too (fail-closed), never silently allowed. + if ! subjects=$(git log --format=%s "$lsha" 2>/dev/null); then + echo "push blocked: could not inspect the commits at $lsha (failing closed)." >&2 + exit 1 + fi + if printf '%s\n' "$subjects" | grep -q '^\[eval-workspace-local\]'; then + echo "push blocked: it contains [eval-workspace-local] plumbing commits (dev shims — never upstream)." >&2 + exit 1 + fi + if printf '%s\n' "$subjects" | grep -q '^\[eval-workspace-upstream\]'; then + echo "push blocked: it contains [eval-workspace-upstream] commits. Upstream those via a clean branch:" >&2 + echo " mise run publish --with " >&2 + exit 1 + fi + done <&2; exit 1; } + for p in $(patches_for "$repo"); do apply_one "$repo" "$p"; done +done +for name in $PUBLISH_REPOS; do + dir=$(repo_dir "$name") + [ -e "$dir/.git" ] && install_pre_push_guard "$dir" +done +echo " pre-push guards installed" diff --git a/workspace/scripts/clone-docs.sh b/workspace/scripts/clone-docs.sh new file mode 100755 index 00000000..e56ee7bd --- /dev/null +++ b/workspace/scripts/clone-docs.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +source scripts/patches-lib.sh +SUPABASE_REMOTE="${SUPABASE_REMOTE:-$(repo_remote supabase)}" + +if [ -e supabase/.git ]; then + echo "supabase/ already cloned; refreshing sparse checkout" +elif [ -e supabase ]; then + echo "ERROR: supabase/ exists but is not a Git clone" >&2 + exit 1 +else + git clone --filter=blob:none --no-checkout "$SUPABASE_REMOTE" supabase + git -C supabase sparse-checkout init --cone +fi + +git -C supabase sparse-checkout set \ + apps/docs \ + examples \ + packages/ai-commands packages/api-types packages/build-icons \ + packages/common packages/config packages/dev-tools \ + packages/eslint-config-supabase packages/icons packages/shared-data \ + packages/tsconfig packages/ui packages/ui-patterns \ + patches supabase + +git -C supabase checkout +pnpm --dir supabase install --filter docs... --frozen-lockfile diff --git a/workspace/scripts/docs-api.sh b/workspace/scripts/docs-api.sh new file mode 100755 index 00000000..671f03f4 --- /dev/null +++ b/workspace/scripts/docs-api.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Serve the docs content GraphQL API locally (standalone adapter) for search_docs. +# Point evals at it: SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql +set -euo pipefail +cd "$(dirname "$0")/.." + +source scripts/load-keys.sh +set -a +source supabase/apps/docs/.env.development +set +a +eval "$(supabase status --workdir supabase -o env)" +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" +NODE_ENV=development \ +NEXT_PUBLIC_SUPABASE_URL="$API_URL" \ +NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" \ +OPENAI_API_KEY="$OPENAI_API_KEY" \ +exec pnpm --dir supabase/apps/docs exec tsx \ + --conditions=react-server \ + --tsconfig tsconfig.json \ + ../../../scripts/docs-content-api.ts diff --git a/workspace/scripts/docs-content-api.ts b/workspace/scripts/docs-content-api.ts new file mode 100644 index 00000000..74dadd83 --- /dev/null +++ b/workspace/scripts/docs-content-api.ts @@ -0,0 +1,36 @@ +import { createServer } from 'node:http' +import { GET, OPTIONS, POST } from '../supabase/apps/docs/app/api/graphql/route.ts' + +const handlers = { GET, OPTIONS, POST } +const port = Number(process.env.PORT ?? 3001) + +createServer(async (incoming, outgoing) => { + const url = new URL( + incoming.url ?? '/', + `http://${incoming.headers.host ?? `127.0.0.1:${port}`}` + ) + const handler = handlers[incoming.method as keyof typeof handlers] + if (url.pathname !== '/docs/api/graphql' || !handler) { + outgoing.writeHead(404).end() + return + } + + const headers = new Headers() + for (const [name, value] of Object.entries(incoming.headers)) { + if (Array.isArray(value)) value.forEach((item) => headers.append(name, item)) + else if (value !== undefined) headers.set(name, value) + } + + const chunks: Buffer[] = [] + for await (const chunk of incoming) chunks.push(Buffer.from(chunk)) + const body = + incoming.method === 'GET' || incoming.method === 'HEAD' + ? undefined + : Buffer.concat(chunks).toString('utf8') + const response = await handler(new Request(url, { method: incoming.method, headers, body })) + + outgoing.writeHead(response.status, Object.fromEntries(response.headers.entries())) + outgoing.end(Buffer.from(await response.arrayBuffer())) +}).listen(port, '127.0.0.1', () => { + console.log(`Docs content API: http://127.0.0.1:${port}/docs/api/graphql`) +}) diff --git a/workspace/scripts/docs-down.sh b/workspace/scripts/docs-down.sh new file mode 100755 index 00000000..cbf7c75f --- /dev/null +++ b/workspace/scripts/docs-down.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +supabase stop --workdir supabase diff --git a/workspace/scripts/docs-index.sh b/workspace/scripts/docs-index.sh new file mode 100755 index 00000000..1646a208 --- /dev/null +++ b/workspace/scripts/docs-index.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +if ! git -C supabase apply --reverse --check "$PWD/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then + echo 'ERROR: fail-closed index patch is not applied; run scripts/apply-patches.sh' >&2 + exit 1 +fi + +source scripts/load-keys.sh +set -a +source supabase/apps/docs/.env.development +set +a +eval "$(supabase status --workdir supabase -o env)" +export NEXT_PUBLIC_SUPABASE_URL="$API_URL" +export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" +export SUPABASE_SECRET_KEY="$SECRET_KEY" +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" +scripts/openai-preflight.sh # fail fast if the key can't run embeddings +# Deliberately allow embedding without prod-only sources (e.g. lint-warnings, +# which needs DOCS_GITHUB_APP_*). Sources gate their own skip on this flag. +export DOCS_EMBED_ALLOW_MISSING_SOURCES=1 + +pnpm --dir supabase/apps/docs run embeddings +node scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-seed.sh b/workspace/scripts/docs-seed.sh new file mode 100755 index 00000000..32809785 --- /dev/null +++ b/workspace/scripts/docs-seed.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +if ! git -C supabase apply --reverse --check "$PWD/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then + echo 'ERROR: fail-closed index patch is not applied; run scripts/apply-patches.sh' >&2 + exit 1 +fi + +source scripts/load-keys.sh +set -a +source supabase/apps/docs/.env.development +set +a +eval "$(supabase status --workdir supabase -o env)" +export NEXT_PUBLIC_SUPABASE_URL="$API_URL" +export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" +export SUPABASE_SECRET_KEY="$SECRET_KEY" +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" +scripts/openai-preflight.sh # fail fast if the key can't run embeddings +# Deliberately allow embedding without prod-only sources (e.g. lint-warnings, +# which needs DOCS_GITHUB_APP_*). Sources gate their own skip on this flag. +export DOCS_EMBED_ALLOW_MISSING_SOURCES=1 + +printf '%s\n' \ + 'Full docs embedding rebuild' \ + 'Model: text-embedding-ada-002 (1536 dimensions)' \ + 'Rate: $0.10 per 1,000,000 input tokens; corpus cost is not known in advance.' +read -r -p "Type 'seed' to spend OpenAI credits and continue: " confirmation +if [ "$confirmation" != seed ]; then + echo 'Seed cancelled.' + exit 1 +fi + +pnpm --dir supabase/apps/docs run embeddings:refresh +node scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-up.sh b/workspace/scripts/docs-up.sh new file mode 100755 index 00000000..af026a0f --- /dev/null +++ b/workspace/scripts/docs-up.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." +supabase start --workdir supabase \ + -x realtime,storage-api,imgproxy,mailpit,postgres-meta,studio,edge-runtime,logflare,vector,supavisor + +# Upstream page migrations grant service_role only Dxt (no CRUD). The new secret +# key authenticates as service_role and bypasses RLS but NOT SQL GRANTs, so the +# embedder (service_role) can't write the content tables. Grant CRUD locally +# (idempotent; native psql is absent on this host, so run it in the db container). +docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -q -c \ + "GRANT ALL ON public.page, public.page_section TO service_role; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO service_role; GRANT SELECT ON public.page, public.page_section TO anon, authenticated;" diff --git a/workspace/scripts/eval.sh b/workspace/scripts/eval.sh new file mode 100755 index 00000000..f9b406d3 --- /dev/null +++ b/workspace/scripts/eval.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Run evals with the workspace env applied. All args pass through to `pnpm eval`. +# API keys come from the macOS keychain if present (see README), else evals' .env. +# e.g. scripts/eval.sh --eval investigate-auth-001 --experiment claude-code-sonnet-5 +set -euo pipefail +cd "$(dirname "$0")/.." + +[ -e evals/.env ] || { echo "evals/.env missing — run: mise run setup" >&2; exit 1; } +source scripts/load-keys.sh +cd evals && pnpm eval "$@" diff --git a/workspace/scripts/hooks.test.sh b/workspace/scripts/hooks.test.sh new file mode 100755 index 00000000..e4ad6445 --- /dev/null +++ b/workspace/scripts/hooks.test.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Self-test for the pre-push guard lifecycle (install / chain / reinstall). +# Uses the mcp clone's hooks dir — .git internals only, repo content untouched; +# every fixture is removed and the real guard is reinstalled at the end. +set -euo pipefail +cd "$(dirname "$0")/.." + +[ -e mcp/.git ] || { echo "SKIP: mcp not cloned"; exit 0; } +HOOKS=$(git -C mcp rev-parse --path-format=absolute --git-path hooks) +HOOK="$HOOKS/pre-push" +CHAINED="$HOOKS/pre-push.eval-workspace-chained" +[ ! -e "$CHAINED" ] || { echo "SKIP: $CHAINED already exists (real chained hook — not touching it)"; exit 0; } + +pass=0; fail=0 +ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $1 (want[$3] got[$2])"; fi; } +cleanup() { rm -f "$CHAINED"; scripts/apply-patches.sh >/dev/null 2>&1 || true; } +trap cleanup EXIT + +# baseline: generated guard present +scripts/apply-patches.sh >/dev/null +ck "guard installed" "$(grep -c 'eval-workspace pre-push guard' "$HOOK")" "1" + +# foreign hook appears -> next install chains it +printf '#!/usr/bin/env bash\n# foreign hook one\nexit 0\n' > "$HOOK"; chmod +x "$HOOK" +scripts/apply-patches.sh >/dev/null 2>&1 +ck "foreign hook chained" "$(grep -c 'foreign hook one' "$CHAINED" 2>/dev/null || echo 0)" "1" +ck "guard reinstalled" "$(grep -c 'eval-workspace pre-push guard' "$HOOK")" "1" + +# a DIFFERENT foreign hook overwrites the generated guard -> reinstall must +# refuse rather than silently clobber the previously chained hook +printf '#!/usr/bin/env bash\n# foreign hook two\nexit 0\n' > "$HOOK"; chmod +x "$HOOK" +rc=$(scripts/apply-patches.sh >/dev/null 2>&1; echo $?) +ck "reinstall refuses to clobber chained" "$rc" "1" +ck "chained hook one preserved" "$(grep -c 'foreign hook one' "$CHAINED")" "1" +ck "foreign hook two preserved" "$(grep -c 'foreign hook two' "$HOOK")" "1" + +# identical foreign hook (e.g. re-copied) -> harmless, install proceeds +cp "$CHAINED" "$HOOK" +rc=$(scripts/apply-patches.sh >/dev/null 2>&1; echo $?) +ck "identical foreign re-chain ok" "$rc" "0" +ck "guard active again" "$(grep -c 'eval-workspace pre-push guard' "$HOOK")" "1" + +echo "hooks.test: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/workspace/scripts/load-keys.sh b/workspace/scripts/load-keys.sh new file mode 100755 index 00000000..bd73799d --- /dev/null +++ b/workspace/scripts/load-keys.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Export API keys from the macOS login keychain into the calling shell, so eval +# runs use them without a plaintext copy in .env. Keys not in the keychain are +# skipped; .env stays the fallback. Exported env beats node's --env-file, so a +# keychain value takes precedence when both exist. +# +# Usage: source scripts/load-keys.sh (the eval recipes do this) +# Running it directly is harmless — it only exports into its own child shell +# and prints nothing, so secrets never reach stdout/scrollback. +# Add a key: mise run store-key ANTHROPIC_API_KEY (hidden prompt; never the bare +# interactive `security -w` — it truncates pasted input at 128 chars) +# Check one: security find-generic-password -a "$USER" -s eval-workspace:OPENAI_API_KEY >/dev/null && echo present + +for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do + if value=$(security find-generic-password -a "$USER" -s "eval-workspace:$key" -w 2>/dev/null); then + export "$key=$value" + fi +done diff --git a/workspace/scripts/manifest.mjs b/workspace/scripts/manifest.mjs new file mode 100755 index 00000000..f3184e3e --- /dev/null +++ b/workspace/scripts/manifest.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +// Loader + CLI for manifest.json — the single source of truth for the repos +// this workspace wires together: checkout dirs, upstream remotes, and the +// ordered enabler-patch set per repo (patch names without the patches/ +// prefix or .patch suffix; "localPatches" marks the dev shims that must never +// reach an upstream PR — see patches/README.md for per-patch provenance). +// +// loadManifest() is THE loader: every consumer (the CLI below, the bash +// facade scripts/patches-lib.sh through it, and provenance.mjs via import) +// goes through the same read+validate, so a schema error fails identically +// everywhere. Never grep the JSON. +// +// CLI usage: +// manifest.mjs repos repo names, one per line, manifest order +// manifest.mjs get value; arrays print one item per line; +// absent optional keys print nothing (exit 0) +// manifest.mjs kind "local" | "upstream" for a patch name +import { readFileSync, realpathSync } from "node:fs"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { dirname, join } from "node:path"; + +const manifestPath = join(dirname(fileURLToPath(import.meta.url)), "..", "manifest.json"); + +/** Read + validate manifest.json. Throws with an actionable message. */ +export function loadManifest() { + let manifest; + try { + manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + } catch (e) { + throw new Error(`cannot read ${manifestPath}: ${e.message}`); + } + + // --- schema validation: fail loud before answering any query --- + const isStr = (v) => typeof v === "string" && v.length > 0; + const isStrArr = (v) => Array.isArray(v) && v.length > 0 && v.every(isStr); + if (typeof manifest?.repos !== "object" || manifest.repos === null || Array.isArray(manifest.repos)) { + throw new Error('top-level "repos" object missing'); + } + for (const [name, repo] of Object.entries(manifest.repos)) { + if (!isStr(repo?.dir)) throw new Error(`repos.${name}.dir must be a non-empty string`); + if (repo.remote !== undefined && !isStr(repo.remote)) throw new Error(`repos.${name}.remote must be a non-empty string`); + if (repo.patches !== undefined && !isStrArr(repo.patches)) throw new Error(`repos.${name}.patches must be a non-empty array of strings`); + if (repo.localPatches !== undefined) { + if (!isStrArr(repo.localPatches)) throw new Error(`repos.${name}.localPatches must be a non-empty array of strings`); + for (const p of repo.localPatches) { + if (!(repo.patches ?? []).includes(p)) throw new Error(`repos.${name}.localPatches has "${p}" which is not in patches`); + } + } + } + // Patch names must be globally unique: `kind` scans localPatches across ALL + // repos while other consumers scan per-repo — a duplicated name would let + // the two silently disagree. Also note "local" is opt-in: a patch absent + // from every localPatches defaults to upstream (publishable via --with). + const seen = new Set(); + for (const [name, repo] of Object.entries(manifest.repos)) { + for (const p of repo.patches ?? []) { + if (seen.has(p)) throw new Error(`patch name "${p}" appears under more than one repo (repos.${name}) — names must be globally unique`); + seen.add(p); + } + } + return manifest; +} + +// --- CLI: runs only when executed directly (not when imported) --- +// Compare REALPATHS: node resolves the ESM main-module URL through symlinks +// (macOS /tmp -> /private/tmp), so a naive pathToFileURL(argv[1]) mismatches. +const isMain = (() => { + try { + return import.meta.url === pathToFileURL(realpathSync(process.argv[1] ?? "")).href; + } catch { + return false; + } +})(); +if (isMain) { + const die = (msg) => { + console.error(`manifest.mjs: ${msg}`); + process.exit(1); + }; + + let manifest; + try { + manifest = loadManifest(); + } catch (e) { + die(e.message); + } + + const [cmd, ...args] = process.argv.slice(2); + switch (cmd) { + case "repos": + console.log(Object.keys(manifest.repos).join("\n")); + break; + case "get": { + const [name, key] = args; + const repo = manifest.repos[name ?? ""]; + if (!repo) die(`unknown repo "${name ?? ""}"`); + if (!["dir", "remote", "patches", "localPatches"].includes(key ?? "")) die(`unknown key "${key ?? ""}"`); + const v = repo[key]; + if (v !== undefined) console.log(Array.isArray(v) ? v.join("\n") : v); + break; + } + case "kind": { + const [patch] = args; + if (typeof patch !== "string" || patch.length === 0) die("kind: patch name required"); + const local = Object.values(manifest.repos).some((r) => (r.localPatches ?? []).includes(patch)); + console.log(local ? "local" : "upstream"); + break; + } + default: + die(`unknown command "${cmd ?? ""}" (expected: repos | get | kind)`); + } +} diff --git a/workspace/scripts/mcp-eval.sh b/workspace/scripts/mcp-eval.sh new file mode 100755 index 00000000..16c8b911 --- /dev/null +++ b/workspace/scripts/mcp-eval.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Run evals against the local mcp build (mise task `mcp-eval` builds it first). +# Add SUPABASE_CONTENT_API_URL= for a local docs index (Phase 2). +set -euo pipefail +cd "$(dirname "$0")/.." +ROOT="$PWD" + +[ -e evals/.env ] || { echo "evals/.env missing — run: mise run setup" >&2; exit 1; } +source scripts/load-keys.sh +cd evals && SUPABASE_MCP_SERVER_PATH="$ROOT/mcp/packages/mcp-server-supabase" pnpm eval "$@" diff --git a/workspace/scripts/openai-preflight.sh b/workspace/scripts/openai-preflight.sh new file mode 100755 index 00000000..654af55b --- /dev/null +++ b/workspace/scripts/openai-preflight.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Fail fast if OPENAI_API_KEY can't run embeddings, with a message specific to +# the failure kind (credential vs quota vs service vs network) so a transient +# outage doesn't send you into a needless key replacement. One ~1-token call, +# effectively free. Reads OPENAI_API_KEY from the environment. Used by +# docs-seed.sh and docs-index.sh. +set -euo pipefail +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + +code=$(curl -s -o /dev/null -w '%{http_code}' \ + --connect-timeout 10 --max-time 30 \ + https://api.openai.com/v1/embeddings \ + -H "Authorization: Bearer $OPENAI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"text-embedding-ada-002","input":"preflight"}') || code="000" + +case "$code" in + 200) ;; + 401|403) + echo "ERROR: OpenAI rejected the key (HTTP $code) — truncated / invalid / wrong scope." >&2 + echo 'Re-add the full key (hidden prompt, truncation-safe, any shell):' >&2 + echo ' mise run store-key OPENAI_API_KEY' >&2 + exit 1 ;; + 429) + echo "ERROR: OpenAI rate/quota limit (HTTP 429) — key is valid but rate-limited or out of quota. Retry later / check billing." >&2 + exit 1 ;; + 5??) + echo "ERROR: OpenAI service error (HTTP $code) — transient, retry shortly. The key is not necessarily bad." >&2 + exit 1 ;; + 000) + echo "ERROR: could not reach api.openai.com (network / DNS / timeout). Check connectivity; the key is not necessarily bad." >&2 + exit 1 ;; + *) + echo "ERROR: unexpected OpenAI response (HTTP $code)." >&2 + exit 1 ;; +esac diff --git a/workspace/scripts/patches-lib.sh b/workspace/scripts/patches-lib.sh new file mode 100644 index 00000000..a0146b78 --- /dev/null +++ b/workspace/scripts/patches-lib.sh @@ -0,0 +1,55 @@ +# Bash facade over manifest.json — the single source of truth for repos, +# remotes, and enabler patches (order and kind; marker subjects derive from +# the kind). All data queries go through scripts/manifest.mjs, which validates +# the schema and fails loud. Sourced by apply-patches.sh, update.sh, +# publish.sh, status.sh, setup.sh, clone-docs.sh. Requires bash (BASH_SOURCE) +# and node (pinned via mise). +_MANIFEST_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +_manifest() { node "$_MANIFEST_LIB_DIR/manifest.mjs" "$@"; } + +# PATCH_REPOS = repos carrying enabler patches (manifest order) +# PUBLISH_REPOS = every publishable editing surface (patch repos + submodules) +# +# Fail-loud init: `for x in $(failing-cmd)` does NOT trip `set -e` in the +# sourcing script — the substitution failure is swallowed and the lists come +# out empty, turning a broken manifest into silent no-ops downstream +# (update.sh would "update" zero repos and exit 0). Capture first, verify. +_MANIFEST_REPOS=$(_manifest repos) || _MANIFEST_REPOS="" +if [ -z "$_MANIFEST_REPOS" ]; then + echo "patches-lib.sh: manifest.mjs returned no repos (broken manifest.json or node failure above) — aborting" >&2 + exit 1 +fi +PATCH_REPOS="" +PUBLISH_REPOS="" +for _r in $_MANIFEST_REPOS; do + PUBLISH_REPOS="${PUBLISH_REPOS:+$PUBLISH_REPOS }$_r" + if [ -n "$(_manifest get "$_r" patches)" ]; then + PATCH_REPOS="${PATCH_REPOS:+$PATCH_REPOS }$_r" + fi +done +unset _r _MANIFEST_REPOS + +repo_dir() { local d; d=$(_manifest get "$1" dir); echo "${d:-$1}"; } +repo_remote() { _manifest get "$1" remote; } + +patches_for() { + local out="" n + for n in $(_manifest get "$1" patches); do + out="${out:+$out }patches/$n.patch" + done + echo "$out" +} + +# local = dev shim, must NEVER reach an upstream PR +# upstream = real fix/feature; `mise run publish --with ` cherry-picks it +# onto a clean PR branch (reworded to drop the marker) +patch_kind() { _manifest kind "$(basename "$1" .patch)"; } + +patch_subject() { + local name; name=$(basename "$1" .patch) + if [ "$(patch_kind "$1")" = local ]; then + echo "[eval-workspace-local] $name" + else + echo "[eval-workspace-upstream] $name" + fi +} diff --git a/workspace/scripts/provenance.mjs b/workspace/scripts/provenance.mjs new file mode 100755 index 00000000..2e3d2960 --- /dev/null +++ b/workspace/scripts/provenance.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node +// Workspace provenance receipt: exact repo SHAs, dirty state (tracked diff +// hash + untracked file content hashes), enabler-patch fingerprints + marker +// presence, and the docs-index stamp. This is the record of WHAT an eval +// actually ran against. +// +// Usage: +// provenance.mjs print the provenance object (JSON) +// provenance.mjs --embed add it as `provenance` to a result JSON +// provenance.mjs --stamp-docs-index write .docs-index-stamp.json describing +// the docs content state just embedded +// (called by docs-index.sh / docs-seed.sh +// after a SUCCESSFUL embed run only) +// +// Consumers: `status.sh --json` and ab.sh (embedded per-arm receipts, captured +// inside run_eval after each arm's sync so baseline/treatment receipts differ +// exactly by the edit under test). Report-only by design: nothing gates on +// these fields yet — receipt shape gets validated on real runs first. +import { readFileSync, writeFileSync, existsSync, rmSync, renameSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { loadManifest } from "./manifest.mjs"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +// The one manifest loader (read + schema validation) — imported, not spawned, +// so this file and the manifest.mjs CLI structurally cannot diverge on how a +// bad manifest fails. +let manifest; +try { + manifest = loadManifest(); +} catch (e) { + console.error(`provenance.mjs: ${e.message}`); + process.exit(1); +} +const STAMP = join(root, ".docs-index-stamp.json"); +// Docs corpus source dir inside the supabase clone (the docs loop's edit scope). +const DOCS_CONTENT_DIR = "apps/docs/content"; +// The embed pipeline's model/dims are read FROM the pipeline source at stamp +// time (no duplicated literals that can silently drift); extraction failure +// fails the stamp closed. +const EMBED_PIPELINE_FILE = "supabase/apps/docs/scripts/search/generate-embeddings.ts"; +const embedPipelineConfig = () => { + const src = readFileSync(join(root, EMBED_PIPELINE_FILE), "utf8"); + const model = src.match(/EMBEDDING_MODEL:\s*'([^']+)'/)?.[1]; + const dims = src.match(/EMBEDDING_DIMENSION:\s*(\d+)/)?.[1]; + if (!model || !dims) { + throw new Error(`cannot extract EMBEDDING_MODEL/EMBEDDING_DIMENSION from ${EMBED_PIPELINE_FILE} — pipeline layout changed?`); + } + return { model, dimensions: Number(dims) }; +}; + +const git = (dir, ...args) => { + try { + return execFileSync("git", ["-C", join(root, dir), ...args], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trimEnd(); + } catch { + return null; + } +}; +// Buffer-returning variant for anything that gets hashed (binary-safe). +const gitRaw = (dir, ...args) => { + try { + return execFileSync("git", ["-C", join(root, dir), ...args], { + stdio: ["ignore", "pipe", "pipe"], + }); + } catch { + return null; + } +}; +const sha256 = (data) => createHash("sha256").update(data).digest("hex"); + +const repoState = (dir) => { + const dirty = (git(dir, "status", "--porcelain") ?? "").length > 0; + // Hash the TRACKED diff only when there is one: an untracked-only dirty + // tree keeps dirty:true but dirty_diff_sha256:null (the untracked list + // below carries that state) — a hash-of-empty-diff here would mislead. + const trackedDiff = dirty ? gitRaw(dir, "diff", "HEAD", "--binary") : null; + // Untracked files are invisible to `git diff HEAD` but are real workspace + // state (e.g. a brand-new guide page) — record path + content hash, sorted. + const untracked = (git(dir, "ls-files", "--others", "--exclude-standard") ?? "") + .split("\n") + .filter(Boolean) + .sort() + .map((p) => { + try { + return { path: p, sha256: sha256(readFileSync(join(root, dir, p))) }; + } catch { + return { path: p, sha256: null }; + } + }); + return { + branch: git(dir, "rev-parse", "--abbrev-ref", "HEAD"), + sha: git(dir, "rev-parse", "HEAD"), + dirty, + dirty_diff_sha256: trackedDiff && trackedDiff.length > 0 ? sha256(trackedDiff) : null, + untracked, + }; +}; + +// Receipt assembly is LAZY: only the paths that emit a receipt (print/--embed) +// pay for it. --stamp-docs-index must not — repoState alone walks and hashes +// every clone's untracked files. +const buildProvenance = () => { + const repos = {}; + const patches = {}; + for (const [name, spec] of Object.entries(manifest.repos)) { + if (!existsSync(join(root, spec.dir, ".git"))) { + repos[name] = { dir: spec.dir, cloned: false }; + continue; + } + repos[name] = { dir: spec.dir, cloned: true, ...repoState(spec.dir) }; + + const localSubjects = (git(spec.dir, "log", "--format=%s", "HEAD", "--not", "--remotes") ?? "").split("\n"); + for (const p of spec.patches ?? []) { + const file = join(root, "patches", `${p}.patch`); + const kind = (spec.localPatches ?? []).includes(p) ? "local" : "upstream"; + patches[p] = { + repo: name, + kind, + patch_sha256: existsSync(file) ? sha256(readFileSync(file)) : null, + marker_present: localSubjects.includes(`[eval-workspace-${kind}] ${p}`), + }; + } + } + + // SUPABASE_MCP_SERVER_PATH swaps the mcp server for an arbitrary local + // build, so a receipt that only records the mcp CLONE's SHA would mislabel + // the arm. Record the override target verbatim plus, when it lives inside a + // git checkout (the workspace clone, the evals submodule, anywhere), that + // checkout's exact HEAD and dirty state. + let mcp_override = null; + const overridePath = process.env.SUPABASE_MCP_SERVER_PATH; + if (overridePath) { + mcp_override = { path: overridePath, checkout_sha: null, checkout_dirty: null }; + try { + // Same entrypoint shapes as the harness override accepts (.js/.mjs/.cjs). + const top = execFileSync( + "git", + ["-C", /\.[cm]?js$/.test(overridePath) ? dirname(overridePath) : overridePath, "rev-parse", "--show-toplevel"], + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ).trimEnd(); + mcp_override.checkout_sha = execFileSync("git", ["-C", top, "rev-parse", "HEAD"], { encoding: "utf8" }).trimEnd(); + mcp_override.checkout_dirty = execFileSync("git", ["-C", top, "status", "--porcelain"], { encoding: "utf8" }).trim().length > 0; + } catch { + // not inside a git checkout (or path missing): the verbatim path still records the arm + } + } + + let docs_index = null; + if (existsSync(STAMP)) { + try { + docs_index = JSON.parse(readFileSync(STAMP, "utf8")); + } catch { + docs_index = { error: "unreadable .docs-index-stamp.json" }; + } + } + + return { generated_at: new Date().toISOString(), repos, patches, mcp_override, docs_index }; +}; + +// Combined fingerprint of the supabase enabler-patch set: part of the docs +// stamp identity, since the patches shape what the embed pipeline does. +const supabasePatchSetSha256 = () => { + const names = manifest.repos.supabase?.patches ?? []; + const h = createHash("sha256"); + for (const p of names) { + const file = join(root, "patches", `${p}.patch`); + if (!existsSync(file)) { + throw new Error(`manifest lists supabase patch "${p}" but ${file} is missing — manifest/patches drift`); + } + h.update(readFileSync(file)); + } + return h.digest("hex"); +}; + +const [cmd, target] = process.argv.slice(2); + +if (cmd === "--stamp-docs-index") { + // The stamp is a REPORT step running after the (paid) embed: a stamp + // failure must never turn a successful docs-index run into a nonzero exit. + // Fail closed on CONTENT (write no stamp, remove any stale one so nothing + // misdescribes the new embed) but exit 0 with a loud warning. + try { + if (!existsSync(join(root, "supabase", ".git"))) { + throw new Error("--stamp-docs-index needs the supabase clone"); + } + const dirtyDiff = gitRaw("supabase", "diff", "HEAD", "--binary", "--", DOCS_CONTENT_DIR); + // Untracked corpus files (a brand-new guide) are part of the embedded state + // but invisible to `git diff HEAD` — record them path + content hash, sorted. + const contentUntracked = (git("supabase", "ls-files", "--others", "--exclude-standard", "--", DOCS_CONTENT_DIR) ?? "") + .split("\n") + .filter(Boolean) + .sort() + .map((p) => { + try { + return { path: p, sha256: sha256(readFileSync(join(root, "supabase", p))) }; + } catch { + return { path: p, sha256: null }; + } + }); + const pipeline = embedPipelineConfig(); + const stamp = { + generated_at: new Date().toISOString(), + // SCOPE: this stamp identifies the REPO-BACKED docs content slice only + // (guides under apps/docs/content). The embed corpus also includes + // non-repo sources (GitHub discussions, generated reference docs, lint + // warnings, partner data) that this stamp cannot capture — and with + // DOCS_EMBED_ALLOW_MISSING_SOURCES, skipped sources leave their prior DB + // rows intact. Do NOT read this as identifying the whole index; that + // needs pipeline-side per-page checksum provenance (upstream fix first). + scope: "repo-docs-content-only", + external_sources_not_captured: true, + supabase_sha: git("supabase", "rev-parse", "HEAD"), + repo_docs_content_state: { + content_tree: git("supabase", "rev-parse", `HEAD:${DOCS_CONTENT_DIR}`), + content_dirty_diff_sha256: dirtyDiff && dirtyDiff.length > 0 ? sha256(dirtyDiff) : null, + content_untracked: contentUntracked, + }, + embedding_model: pipeline.model, + embedding_dimensions: pipeline.dimensions, + patch_set_sha256: supabasePatchSetSha256(), + // schema_version deliberately omitted: there is no truthful source for it + // yet (the content-DB migration state isn't stamped by the pipeline). + }; + writeFileSync(STAMP, JSON.stringify(stamp, null, 2) + "\n"); + console.log(`stamped ${STAMP}`); + } catch (e) { + rmSync(STAMP, { force: true }); + console.error( + `provenance.mjs: WARNING — docs-index stamp NOT written (${e.message}); ` + + `any previous stamp was removed so receipts show docs_index: null instead of a stale claim. ` + + `The embed itself is unaffected.`, + ); + } + process.exit(0); +} + + +if (cmd === "--embed") { + if (!target) { + console.error("provenance.mjs: --embed needs a result-file path"); + process.exit(1); + } + let result; + try { + result = JSON.parse(readFileSync(target, "utf8")); + } catch (e) { + console.error(`provenance.mjs: --embed cannot read result JSON at ${target}: ${e.message}`); + process.exit(1); + } + result.provenance = buildProvenance(); + // Same-directory temp + rename: an interrupt mid-write cannot leave a + // truncated result file behind. + const tmp = `${target}.tmp-${process.pid}`; + writeFileSync(tmp, JSON.stringify(result, null, 2) + "\n"); + renameSync(tmp, target); +} else if (cmd === undefined) { + console.log(JSON.stringify(buildProvenance(), null, 2)); +} else { + console.error(`provenance.mjs: unknown arg "${cmd}" (expected: nothing | --embed | --stamp-docs-index)`); + process.exit(1); +} diff --git a/workspace/scripts/publish.sh b/workspace/scripts/publish.sh new file mode 100755 index 00000000..e6dbcd06 --- /dev/null +++ b/workspace/scripts/publish.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Build a CLEAN branch for an upstream PR, in a temporary worktree that starts +# at origin/ and contains ONLY what you select: +# - your own commits (everything above origin that isn't [eval-workspace-*]) +# - optionally, UPSTREAM-CANDIDATE plumbing commits via --with , +# reworded to drop the marker so the PR is clean +# [eval-workspace-local] commits can never be selected, and the pre-push guard +# blocks any marker commit from being pushed by accident. +# +# Usage: +# scripts/publish.sh --list what's publishable +# scripts/publish.sh [--with ]... +# : evals | mcp | supabase | skills (the agent-skills submodule) +set -euo pipefail +cd "$(dirname "$0")/.." +ROOT="$PWD" +source scripts/patches-lib.sh + +repo="${1:-}"; shift || true +case " $PUBLISH_REPOS " in *" $repo "*) ;; *) + echo "usage: mise run publish [--with ]... | --list" >&2; exit 2 ;; +esac +dir=$(repo_dir "$repo") +[ -e "$dir/.git" ] || { echo "$dir not cloned" >&2; exit 1; } +branch=$(git -C "$dir" rev-parse --abbrev-ref HEAD) +[ "$branch" != HEAD ] || { echo "$dir is on a detached HEAD — check out a branch first" >&2; exit 1; } + +user_commits() { # oldest first, hash + subject, excluding plumbing markers + git -C "$dir" log --reverse --format='%H %s' "origin/$branch..HEAD" \ + | grep -v ' \[eval-workspace-' || true +} +candidate_sha() { # sha of an [eval-workspace-upstream] commit by patch name + git -C "$dir" log --format='%H %s' "origin/$branch..HEAD" \ + | grep -F "[eval-workspace-upstream] $1" | head -1 | cut -d' ' -f1 +} + +if [ "${1:-}" = "--list" ] || [ $# -eq 0 ]; then + echo "== $repo: publishable from $branch (base origin/$branch) ==" + echo "your commits:" + user_commits | sed 's/^\([0-9a-f]\{8\}\)[0-9a-f]*/ \1/' || true + [ -n "$(user_commits)" ] || echo " (none)" + if [ -n "$(patches_for "$repo")" ]; then + echo "upstream-candidate plumbing (add with --with ):" + for p in $(patches_for "$repo"); do + [ "$(patch_kind "$p")" = upstream ] || continue + echo " $(basename "$p" .patch)" + done + fi + echo + echo "then: mise run publish $repo (--all | --commit ...) [--with ]..." + exit 0 +fi + +topic="$1"; shift +case "$topic" in --*) echo "topic branch must come before the selection flags" >&2; exit 2 ;; esac +ALL=0; COMMITS=(); WITH=() +while [ $# -gt 0 ]; do + case "$1" in + --all) ALL=1; shift ;; + --commit) shift; [ -n "${1:-}" ] || { echo "--commit needs a sha" >&2; exit 2; }; COMMITS+=("$1"); shift ;; + --with) shift; [ -n "${1:-}" ] || { echo "--with needs a patch name" >&2; exit 2; }; WITH+=("$1"); shift ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +# resolve selections -> SHAs. Commit selection is EXPLICIT: --all takes every +# commit of yours; --commit picks specific ones — so stacked, unrelated +# experiments can't ride along into a PR by accident. +PICKS=() +for w in ${WITH[@]+"${WITH[@]}"}; do + sha=$(candidate_sha "$w") + [ -n "$sha" ] || { echo "no [eval-workspace-upstream] commit for '$w' in $dir (see: mise run publish $repo --list)" >&2; exit 1; } + PICKS+=("$sha") +done +if [ "$ALL" = 1 ]; then + [ ${#COMMITS[@]} -eq 0 ] 2>/dev/null || { echo "use --all or --commit, not both" >&2; exit 2; } + while read -r sha _; do [ -n "$sha" ] && PICKS+=("$sha"); done </dev/null; then + for c in ${COMMITS[@]+"${COMMITS[@]}"}; do + sha=$(git -C "$dir" rev-parse --verify --quiet "$c^{commit}") \ + || { echo "not a commit in $dir: $c" >&2; exit 1; } + user_commits | grep -q "^$sha " \ + || { echo "$c is not one of YOUR commits above origin/$branch (see: mise run publish $repo --list)" >&2; exit 1; } + PICKS+=("$sha") + done +fi +if [ ${#PICKS[@]} -eq 0 ] 2>/dev/null; then + echo "nothing selected. Pick commits explicitly:" >&2 + echo " mise run publish $repo $topic --all # every commit of yours" >&2 + echo " mise run publish $repo $topic --commit ... # specific commits" >&2 + echo " mise run publish $repo $topic --with # an upstream-candidate enabler" >&2 + echo >&2 + echo "your commits (mise run publish $repo --list):" >&2 + user_commits | sed 's/^/ /' >&2 + exit 2 +fi +# chronological order (parents first) +ORDERED=() +while read -r sha; do + for p in ${PICKS[@]+"${PICKS[@]}"}; do [ "$sha" = "$p" ] && ORDERED+=("$sha"); done +done <&2; exit 1; } +git -C "$dir" worktree add -q -b "$topic" "$wt" "origin/$branch" \ + || { echo "could not create worktree/branch '$topic' (does the branch already exist?)" >&2; exit 1; } + +fail() { + git -C "$wt" cherry-pick --abort 2>/dev/null || true + git -C "$dir" worktree remove --force "$wt" 2>/dev/null || true + git -C "$dir" branch -D "$topic" 2>/dev/null || true + echo "publish aborted: $1" >&2 + exit 1 +} +for sha in ${ORDERED[@]+"${ORDERED[@]}"}; do + git -C "$wt" cherry-pick "$sha" >/dev/null || fail "cherry-pick conflict at $(git -C "$dir" log -1 --format='%h %s' "$sha")" + subj=$(git -C "$wt" log -1 --format=%s) + case "$subj" in + "[eval-workspace-upstream] "*) # reword: drop the marker + plumbing boilerplate + git -C "$wt" log -1 --format=%B \ + | sed -e '1s/^\[eval-workspace-upstream\] //' -e '/eval-workspace plumbing/d' \ + | git -C "$wt" commit -q --amend -F - ;; + esac +done + +echo "== clean PR branch ready ==" +git -C "$wt" log --oneline "origin/$branch..HEAD" | sed 's/^/ /' +echo +echo "Review and push:" +echo " cd $wt" +echo " git push origin HEAD # then open the PR" +echo +echo "When merged (or abandoned), clean up:" +echo " git -C $dir worktree remove $wt && git -C $dir branch -D $topic" diff --git a/workspace/scripts/setup.sh b/workspace/scripts/setup.sh new file mode 100755 index 00000000..100a21ba --- /dev/null +++ b/workspace/scripts/setup.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Clone + wire the repos needed for the skills loop, install deps, print status. +# Idempotent: existing clones are left alone (pull them yourself). +set -euo pipefail + +cd "$(dirname "$0")/.." +source scripts/patches-lib.sh + +EVALS_REMOTE="${EVALS_REMOTE:-$(repo_remote evals)}" + +log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33mwarn:\033[0m %s\n' "$*" >&2; } + +# --- required tooling --- +for bin in git pnpm; do + command -v "$bin" >/dev/null || { warn "missing required tool: $bin"; exit 1; } +done +if command -v docker >/dev/null; then + docker info >/dev/null 2>&1 || warn "docker installed but not running — start it before sandbox-runtime evals." +else + warn "docker not found — sandbox-runtime evals need it (tools-mode evals don't)." +fi + +# --- .env --- +if [ -f .env ]; then + log ".env already exists — leaving it." +else + cp .env.example .env + log "Created .env from .env.example — fill in your API keys." +fi + +# --- clone evals (with agent-skills submodule) --- +if [ -e evals/.git ]; then + log "evals/ already cloned — skipping." +else + log "Cloning evals (with agent-skills submodule)…" + git clone --recurse-submodules "$EVALS_REMOTE" evals +fi + +# Self-heal: ensure the agent-skills submodule is present even if evals was +# cloned earlier without --recurse-submodules (or a clone was interrupted). +log "Syncing submodules…" +git -C evals submodule update --init --recursive + +# --- share one .env with evals --- +ln -sfn ../.env evals/.env +log "Symlinked .env → evals/.env" + +# --- install deps --- +log "Installing evals deps (pnpm install)…" +( cd evals && pnpm install ) + +# --- enabler plumbing: marker commits + pre-push guards (idempotent) --- +scripts/apply-patches.sh + +log "Setup done." +echo +scripts/status.sh diff --git a/workspace/scripts/status.sh b/workspace/scripts/status.sh new file mode 100755 index 00000000..3ea35ed5 --- /dev/null +++ b/workspace/scripts/status.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Workspace status: per-repo branch/SHA/dirty, env keys present, tooling. +set -euo pipefail + +cd "$(dirname "$0")/.." + +# --json: machine-readable provenance receipt (repos, patches, docs stamp) +if [ "${1:-}" = "--json" ]; then exec node scripts/provenance.mjs; fi +source scripts/patches-lib.sh + +# label, path. `.git` is a dir for clones and a file for submodules, so -e covers both. +repo_status() { + local label="$1" dir="$2" branch sha dirty + if [ ! -e "$dir/.git" ]; then + printf ' %-22s not cloned\n' "$label" + return + fi + branch=$(git -C "$dir" rev-parse --abbrev-ref HEAD 2>/dev/null || echo '?') + sha=$(git -C "$dir" rev-parse --short HEAD 2>/dev/null || echo '?') + if [ -n "$(git -C "$dir" status --porcelain 2>/dev/null)" ]; then dirty=dirty; else dirty=clean; fi + printf ' %-22s %-22s %-10s %s\n' "$label" "$branch" "$sha" "$dirty" +} + +NEXT="" +echo "Repos:" +# Rows come from the manifest (PUBLISH_REPOS order); agent-skills is the one +# hand-nested row, shown under its host clone evals. +for _repo in $PUBLISH_REPOS; do + [ "$_repo" = skills ] && continue + repo_status "$_repo" "$(repo_dir "$_repo")" + if [ "$_repo" = evals ]; then + repo_status " - agent-skills" "$(repo_dir skills)" + fi +done +unset _repo +# setup.sh also repairs a missing agent-skills submodule, root .env, and the +# evals/.env symlink — suggest it if ANY of those is missing (not just the clone). +if [ ! -e evals/.git ] || [ ! -e evals/submodules/agent-skills/.git ] || [ ! -f .env ] || [ ! -e evals/.env ]; then + NEXT="${NEXT} mise run setup # clone/repair evals (+ agent-skills), install, wire .env\n" +fi +# enabler plumbing present? (marker commits; see apply-patches.sh) +for _r in $PATCH_REPOS; do + [ -e "$_r/.git" ] || continue + _subjects=$(git -C "$_r" log --format=%s HEAD --not --remotes 2>/dev/null || true) + for _p in $(patches_for "$_r"); do + if ! printf '%s\n' "$_subjects" | grep -qxF "$(patch_subject "$_p")"; then + NEXT="${NEXT} mise run apply-patches # enabler plumbing commit(s) missing in $_r\n" + break + fi + done +done +# pre-push guards active? (apply-patches installs them) +for _n in $PUBLISH_REPOS; do + _d=$(repo_dir "$_n") + [ -e "$_d/.git" ] || continue + _hooks=$(git -C "$_d" rev-parse --path-format=absolute --git-path hooks 2>/dev/null || true) + if [ -n "$_hooks" ] && ! grep -q 'eval-workspace pre-push guard' "$_hooks/pre-push" 2>/dev/null; then + NEXT="${NEXT} mise run apply-patches # pre-push guard missing in $_d\n" + fi +done + +echo +echo "Credentials (source per key; ANTHROPIC + OPENAI required, GEMINI optional):" +if [ ! -e evals/.env ]; then + echo " ! evals/.env missing — node --env-file will fail; run: mise run setup" +fi +IS_DARWIN=""; [ "$(uname -s)" = Darwin ] && IS_DARWIN=1 +for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do + src="" + [ -n "$IS_DARWIN" ] && security find-generic-password -a "$USER" -s "eval-workspace:$key" >/dev/null 2>&1 && src="keychain" + if [ -f .env ] && grep -qE "^${key}=.+" .env; then src="${src:+$src, }.env"; fi + printf ' %-24s %s\n' "$key" "${src:-MISSING}" + if [ -z "$src" ] && [ "$key" != GEMINI_API_KEY ]; then + if [ -n "$IS_DARWIN" ]; then + NEXT="${NEXT} mise run store-key $key\n" + else + NEXT="${NEXT} add ${key}=… to .env # git-ignored; no macOS keychain here (see README \"Credentials\")\n" + fi + fi +done + +echo +echo "Tooling:" +for bin in mise pnpm docker node; do + if command -v "$bin" >/dev/null; then printf ' %-7s %s\n' "$bin" "$(command -v "$bin")"; else printf ' %-7s missing\n' "$bin"; fi +done +if command -v docker >/dev/null; then + if docker info >/dev/null 2>&1; then echo " docker (running)"; else echo " docker (not running)"; fi +fi + +echo +if [ -n "$NEXT" ]; then + echo "Next (in order):" + printf "%b" "$NEXT" | awk '!seen[$0]++' +else + echo "Ready. Try:" + echo " mise run ab-test # zero-cost self-check of the A/B runner" + echo " mise run ab # A/B readiness probe (head-to-head evals)" + echo " mise run eval -- --eval --experiment # run an eval (see evals/evals/)" +fi diff --git a/workspace/scripts/status.test.sh b/workspace/scripts/status.test.sh new file mode 100755 index 00000000..d809357f --- /dev/null +++ b/workspace/scripts/status.test.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Self-test for status.sh's Next/Ready diagnosis. Runs the WORKING-TREE script +# against synthetic workspace states in a temp dir (never touches this one). +# USER=nobody forces keychain misses so results don't depend on real keys. +set -euo pipefail +cd "$(dirname "$0")/.." +SRC="$PWD" + +pass=0; fail=0 +ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $1 (want[$3] got[$2])"; fi; } + +run_case() { # $1 = label, $2 = state-builder function; output -> /tmp/status_case.out + local tmp; tmp="$(mktemp -d /tmp/eval-ws-status-XXXXXX)" + mkdir -p "$tmp/scripts" "$tmp/bin" + cp "$SRC/scripts/status.sh" "$SRC/scripts/patches-lib.sh" "$SRC/scripts/manifest.mjs" "$tmp/scripts/" + cp "$SRC/manifest.json" "$SRC/.env.example" "$tmp/" + # OS-deterministic: default to Darwin regardless of host; a case may overwrite bin/uname. + printf '#!/bin/sh\necho Darwin\n' > "$tmp/bin/uname"; chmod +x "$tmp/bin/uname" + ( cd "$tmp" && "$2" ) + ( cd "$tmp" && USER=nobody PATH="$PWD/bin:$PATH" bash scripts/status.sh 2>/dev/null ) > /tmp/status_case.out + rm -rf "$tmp" +} +has() { grep -qF -- "$1" /tmp/status_case.out && echo y || echo n; } + +# --- fresh clone: nothing set up --- +state_fresh() { :; } +run_case fresh state_fresh +ck "fresh: suggests setup" "$(has 'mise run setup')" y +ck "fresh: suggests required keys" "$(has 'mise run store-key OPENAI_API_KEY')" y +ck "fresh: not Ready" "$(has 'Ready.')" n + +# --- partial: evals cloned, submodule + evals/.env missing (interrupted setup) --- +state_partial_submodule() { mkdir -p evals/.git; cp .env.example .env; } +run_case partial-submodule state_partial_submodule +ck "partial submodule: suggests setup" "$(has 'mise run setup')" y +ck "partial submodule: not Ready" "$(has 'Ready.')" n + +# --- partial: clone + submodule fine, evals/.env symlink missing --- +state_partial_env() { mkdir -p evals/.git evals/submodules/agent-skills/.git; cp .env.example .env; } +run_case partial-env state_partial_env +ck "partial env: suggests setup" "$(has 'mise run setup')" y +ck "partial env: not Ready" "$(has 'Ready.')" n + +# --- complete workspace, only keys missing: setup NOT suggested, keys are --- +state_keys_only() { mkdir -p evals/.git evals/submodules/agent-skills/.git; cp .env.example .env; ln -s ../.env evals/.env; } +run_case keys-only state_keys_only +ck "keys-only: no setup suggestion" "$(has 'mise run setup')" n +ck "keys-only: suggests keys" "$(has 'mise run store-key ANTHROPIC_API_KEY')" y +ck "keys-only: not Ready" "$(has 'Ready.')" n + +# --- non-Darwin: same complete-but-keyless state; keys must route to .env, never store-key --- +state_linux_keys() { state_keys_only; printf '#!/bin/sh\necho Linux\n' > bin/uname; } +run_case linux-keys state_linux_keys +ck "linux: no store-key suggestion" "$(has 'mise run store-key')" n +ck "linux: routes keys to .env" "$(has 'add ANTHROPIC_API_KEY=')" y +ck "linux: not Ready" "$(has 'Ready.')" n + +echo "status.test: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/workspace/scripts/store-key.sh b/workspace/scripts/store-key.sh new file mode 100755 index 00000000..535a9ae6 --- /dev/null +++ b/workspace/scripts/store-key.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Store an API key in the macOS keychain (item eval-workspace:), reading the +# value with a hidden prompt. Run via `mise run store-key ` — the bash +# shebang makes it work no matter the caller's shell (zsh's `read -p` differs). +# +# Why not the interactive `security -w` prompt: it silently truncates pasted +# input at 128 chars, producing invalid keys. Reading into a variable and +# passing it as an argument is immune, and the value never reaches shell +# history or stdout. +set -euo pipefail + +KEY="${1:-}" +case "$KEY" in + ANTHROPIC_API_KEY|OPENAI_API_KEY|GEMINI_API_KEY) ;; + *) echo "usage: mise run store-key " >&2; exit 2 ;; +esac + +if [ "$(uname -s)" != Darwin ]; then + echo "store-key uses the macOS keychain — not available on this platform." >&2 + echo "Add ${KEY}= to the git-ignored .env instead (see README \"Credentials\")." >&2 + exit 2 +fi + +read -rsp "$KEY (input hidden): " value; echo +[ -n "$value" ] || { echo "empty value — nothing stored" >&2; exit 1; } +len=${#value} +security add-generic-password -U -a "$USER" -s "eval-workspace:$KEY" -w "$value" +unset value +echo "stored eval-workspace:$KEY ($len chars)" diff --git a/workspace/scripts/update.sh b/workspace/scripts/update.sh new file mode 100755 index 00000000..d2748100 --- /dev/null +++ b/workspace/scripts/update.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Update the clones: fetch upstream and REBASE local work — the [eval-workspace-*] +# plumbing commits plus any commits of yours — onto the new tip. Uncommitted +# edits survive via --autostash. A rebase conflict means upstream drifted under +# a plumbing commit (regenerate that patch; see patches/README.md) or under +# your own commits; the rebase is aborted so the workspace is left as found. +# +# Usage: scripts/update.sh [--check] [repo...] +# --check fetch + report how far behind each clone is; changes nothing +# repos default: every patch-carrying repo from manifest.json +set -euo pipefail +cd "$(dirname "$0")/.." +source scripts/patches-lib.sh + +CHECK=0 +[ "${1:-}" = "--check" ] && { CHECK=1; shift; } +REPOS=("$@") +[ ${#REPOS[@]} -gt 0 ] 2>/dev/null || REPOS=($PATCH_REPOS) + +FAILED=0 +for repo in ${REPOS[@]+"${REPOS[@]}"}; do + if [ ! -e "$repo/.git" ]; then echo "== $repo: not cloned — skipping"; continue; fi + branch=$(git -C "$repo" rev-parse --abbrev-ref HEAD) + old_upstream=$(git -C "$repo" rev-parse "origin/$branch" 2>/dev/null || echo "") + git -C "$repo" fetch -q + behind=$(git -C "$repo" rev-list --count "HEAD..origin/$branch" 2>/dev/null || echo '?') + + if [ "$CHECK" = 1 ]; then + printf '== %-9s %s@%s is %s commit(s) behind origin/%s\n' "$repo:" "$branch" "$(git -C "$repo" rev-parse --short HEAD)" "$behind" "$branch" + continue + fi + + echo "== $repo: $branch, $behind commit(s) behind — updating" + git -C "$repo" diff --cached --quiet --ita-visible-in-index \ + || { echo " $repo index has staged changes — unstage or commit first" >&2; FAILED=1; continue; } + + if [ "$behind" = 0 ]; then + echo " already up to date" + continue + fi + + # capture agent-skills state BEFORE the rebase moves the recorded pin + if [ "$repo" = evals ]; then + sk=evals/submodules/agent-skills + old_pin=$(git -C evals ls-tree HEAD submodules/agent-skills | awk '{print $3}') + sk_head=$(git -C "$sk" rev-parse HEAD 2>/dev/null || echo "") + sk_clean=$([ -e "$sk/.git" ] && [ -z "$(git -C "$sk" status --porcelain 2>/dev/null)" ] && echo yes || echo no) + fi + + if ! git -C "$repo" rebase --autostash -q "origin/$branch"; then + git -C "$repo" rebase --abort 2>/dev/null || true + echo " REBASE CONFLICT: upstream drifted under a plumbing commit or your work — nothing changed." >&2 + echo " Fix: regenerate the conflicting patch (patches/README.md) or rebase manually in $repo/." >&2 + FAILED=1 + continue + fi + echo " now at $(git -C "$repo" rev-parse --short HEAD) (upstream $(git -C "$repo" rev-parse --short "origin/$branch"))" + + if [ "$repo" = evals ] && [ -e "$sk/.git" ]; then + # sync the pin only when skills had NO local work before the update: + # clean tree AND sitting exactly on the previously recorded gitlink + if [ "$sk_clean" = yes ] && [ "$sk_head" = "$old_pin" ]; then + git -C evals submodule update --init --recursive -q + else + echo " note: agent-skills submodule has local work (edits or commits) — skipping submodule sync" + fi + fi + + new_upstream=$(git -C "$repo" rev-parse "origin/$branch") + if [ "$old_upstream" != "$new_upstream" ]; then + case "$repo" in + evals) + if ! git -C evals diff --quiet "$old_upstream" "$new_upstream" -- pnpm-lock.yaml; then + echo " lockfile changed — pnpm install"; ( cd evals && pnpm install --silent ) + fi ;; + supabase) + if ! git -C supabase diff --quiet "$old_upstream" "$new_upstream" -- pnpm-lock.yaml; then + echo " lockfile changed — pnpm install (docs filter)"; pnpm --dir supabase install --filter docs... --silent + fi + if [ -n "$(git -C supabase diff --name-only "$old_upstream" "$new_upstream" -- apps/docs/content | head -1)" ]; then + echo " note: docs content changed upstream — the next docs-index re-embeds changed pages (costs cents)" + fi ;; + mcp) + if [ -d mcp/packages/mcp-server-supabase/dist ]; then + echo " rebuilding local mcp"; ( cd mcp && pnpm install --silent && pnpm build ) + fi ;; + esac + fi +done + +[ "$FAILED" = 0 ] || { echo; echo "update finished with errors (see above)"; exit 1; } From 0babcdcb54c3ce5780e72ff6cb71d513d46ef2ba Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 09:49:31 +0200 Subject: [PATCH 3/9] refactor: re-root the workspace glue to live inside evals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit evals is the host now, not a managed clone: every evals/ path prefix drops, 'cd evals && pnpm eval' becomes a root pnpm eval, and the evals manifest entry + marker plumbing die. The mcp clone is replaced by the pinned submodules/mcp working tree (patches apply there as marker commits; submodule.submodules/mcp.ignore=all keeps that patch dirt out of host status/commits — pin bumps are a deliberate git add). supabase stays the opt-in sparse clone. Keychain service name is unchanged so existing stored keys keep working. Self-tests from the repo root: status.test 19/0, hooks.test 8/0 (real install/chain/reinstall lifecycle against the initialized mcp submodule), ab.test 22/0. --- .gitmodules | 1 + workspace/scripts/ab-demo.sh | 13 ++-- workspace/scripts/ab-ready.sh | 16 ++--- workspace/scripts/ab-task.sh | 10 ++-- workspace/scripts/ab.sh | 38 ++++++------ workspace/scripts/ab.test.sh | 58 +++++++++--------- workspace/scripts/affected-task.sh | 4 +- workspace/scripts/affected.ts | 25 ++++---- workspace/scripts/apply-patches.sh | 64 ++++++++++---------- workspace/scripts/clone-docs.sh | 4 +- workspace/scripts/docs-api.sh | 6 +- workspace/scripts/docs-content-api.ts | 2 +- workspace/scripts/docs-down.sh | 2 +- workspace/scripts/docs-index.sh | 12 ++-- workspace/scripts/docs-seed.sh | 12 ++-- workspace/scripts/docs-up.sh | 2 +- workspace/scripts/eval.sh | 12 ++-- workspace/scripts/hooks.test.sh | 81 +++++++++++++++---------- workspace/scripts/manifest.mjs | 25 ++++++-- workspace/scripts/mcp-eval.sh | 7 +-- workspace/scripts/patches-lib.sh | 12 ++-- workspace/scripts/provenance.mjs | 58 ++++++++++++++---- workspace/scripts/publish.sh | 12 ++-- workspace/scripts/setup.sh | 54 +++++++---------- workspace/scripts/status.sh | 58 +++++++++--------- workspace/scripts/status.test.sh | 66 +++++++++++--------- workspace/scripts/update.sh | 86 +++++++++++---------------- 27 files changed, 397 insertions(+), 343 deletions(-) diff --git a/.gitmodules b/.gitmodules index 094026de..f3863093 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,4 @@ [submodule "submodules/mcp"] path = submodules/mcp url = git@github.com:supabase/mcp.git + ignore = all diff --git a/workspace/scripts/ab-demo.sh b/workspace/scripts/ab-demo.sh index d61cdff9..df73f3ee 100755 --- a/workspace/scripts/ab-demo.sh +++ b/workspace/scripts/ab-demo.sh @@ -6,19 +6,19 @@ # `@supabase/pinniped` for the fictional "Nimbus" runtime (append-only edit) # 2. installs a matching throwaway eval (demo/canary-eval/) that PASSES only # if the agent names that package -# 3. runs the real head-to-head (scripts/ab.sh): treatment (fact embedded in -# the local index) vs baseline (fact removed + re-embedded) +# 3. runs the real head-to-head (workspace/scripts/ab.sh): treatment (fact +# embedded in the local index) vs baseline (fact removed + re-embedded) # 4. expected result: baseline FAIL 0/1 -> treatment PASS 1/1 — proof that a # local docs edit alone moves an eval through search_docs # 5. cleans up completely: guide reverted, canary de-embedded, eval removed # # Cost: 2 model runs (claude-sonnet-5) + a few embedding cents. Asks first. set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." GUIDE=supabase/apps/docs/content/guides/auth/choosing-a-server-package.mdx EVAL_ID=investigate-workspace-canary-nimbus-package -EVAL_DST=evals/evals/$EVAL_ID +EVAL_DST=evals/$EVAL_ID fail() { echo "not ready: $1" >&2; echo "run \`mise run ab\` (no args) for the readiness probe + fix commands" >&2; exit 1; } @@ -29,7 +29,6 @@ for k in ANTHROPIC_API_KEY OPENAI_API_KEY; do done # --- hard preflight (same gates the A/B itself needs) --- -[ -e evals/.git ] || fail "evals not cloned" [ -e supabase/.git ] || fail "supabase not cloned" [ -n "${ANTHROPIC_API_KEY:-}" ] || fail "ANTHROPIC_API_KEY missing" [ -n "${OPENAI_API_KEY:-}" ] || fail "OPENAI_API_KEY missing" @@ -49,7 +48,7 @@ cleanup() { echo "== demo cleanup: reverting canary, de-embedding, removing throwaway eval ==" git -C supabase checkout -- "${GUIDE#supabase/}" 2>/dev/null || true rm -rf "$EVAL_DST" - scripts/docs-index.sh >/dev/null 2>&1 \ + workspace/scripts/docs-index.sh >/dev/null 2>&1 \ || echo "WARNING: cleanup re-embed failed — run \`mise run docs-index\` to de-embed the canary" >&2 } trap cleanup EXIT @@ -64,7 +63,7 @@ mkdir -p "$EVAL_DST" cp demo/canary-eval/PROMPT.md demo/canary-eval/EVAL.ts "$EVAL_DST/" echo "== canary planted in $GUIDE; running the head-to-head ==" -scripts/ab.sh "$EVAL_ID" claude-sonnet-5 "$GUIDE" +workspace/scripts/ab.sh "$EVAL_ID" claude-sonnet-5 "$GUIDE" echo echo "What you just saw: the ONLY difference between the two runs was that one" diff --git a/workspace/scripts/ab-ready.sh b/workspace/scripts/ab-ready.sh index e9742e44..9b375f68 100755 --- a/workspace/scripts/ab-ready.sh +++ b/workspace/scripts/ab-ready.sh @@ -3,7 +3,7 @@ # missing, and the exact command to fix each gap. `mise run ab` with no args # lands here. Read-only; never mutates anything. set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." NEXT="" ok() { printf ' [ok] %s\n' "$1"; } @@ -13,16 +13,16 @@ have_key() { security find-generic-password -a "$USER" -s "eval-workspace:$1" >/ echo "A/B readiness — mise run ab [experiment=claude-sonnet-5]" echo -echo "skills loop (edit evals/submodules/agent-skills/skills/…):" -if [ -e evals/submodules/agent-skills/.git ]; then ok "evals + agent-skills cloned"; else miss "evals/agent-skills not cloned" "mise run setup"; fi +echo "skills loop (edit submodules/agent-skills/skills/…):" +if [ -e submodules/agent-skills/.git ]; then ok "agent-skills submodule initialized"; else miss "agent-skills submodule not initialized" "mise run setup"; fi if have_key ANTHROPIC_API_KEY; then ok "ANTHROPIC_API_KEY present"; else miss "ANTHROPIC_API_KEY missing" "mise run store-key ANTHROPIC_API_KEY"; fi echo -echo "mcp loop (edit mcp/packages/…):" -if [ -e mcp/.git ] && [ -d mcp/packages/mcp-server-supabase/dist ]; then - ok "local mcp cloned + built" +echo "mcp loop (edit submodules/mcp/packages/…):" +if [ -e submodules/mcp/.git ] && [ -d submodules/mcp/packages/mcp-server-supabase/dist ]; then + ok "local mcp submodule initialized + built" else - miss "local mcp not cloned/built" "mise run mcp-build" + miss "local mcp submodule not initialized/built" "mise run mcp-build" fi echo @@ -46,6 +46,6 @@ else echo "All loops ready." fi echo -echo "Then: make ONE edit (tracked file, unstaged), pick an eval (ls evals/evals/), run:" +echo "Then: make ONE edit (tracked file, unstaged), pick an eval (ls evals/), run:" echo " mise run ab " echo "Wiring self-test (free): mise run ab-test · guided live demo: mise run ab-demo" diff --git a/workspace/scripts/ab-task.sh b/workspace/scripts/ab-task.sh index 7252140a..7861cc41 100755 --- a/workspace/scripts/ab-task.sh +++ b/workspace/scripts/ab-task.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # mise task wrapper: ab [experiment=claude-sonnet-5]. # No args -> the readiness probe (which loops are runnable + how to fix gaps). -# Reorders to scripts/ab.sh's . For multiple edited -# files or other advanced use, call scripts/ab.sh directly. +# Reorders to ab.sh's . For multiple edited +# files or other advanced use, call workspace/scripts/ab.sh directly. set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." -[ $# -gt 0 ] || exec scripts/ab-ready.sh +[ $# -gt 0 ] || exec workspace/scripts/ab-ready.sh [ $# -ge 2 ] || { echo "usage: mise run ab [experiment] (no args = readiness probe)" >&2; exit 2; } -exec scripts/ab.sh "$1" "${3:-claude-sonnet-5}" "$2" +exec workspace/scripts/ab.sh "$1" "${3:-claude-sonnet-5}" "$2" diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh index d951edc7..462811b5 100755 --- a/workspace/scripts/ab.sh +++ b/workspace/scripts/ab.sh @@ -2,12 +2,12 @@ # Head-to-head eval: baseline (your edit reverted) vs treatment (edit applied). # Same eval, same experiment; the ONLY difference is your uncommitted edit. # -# Usage: scripts/ab.sh [more-paths...] +# Usage: workspace/scripts/ab.sh [more-paths...] # is a file inside a clone; its clone selects the loop + how to # re-sync between states: -# supabase/apps/docs/content/… docs loop (re-embed via docs-index; needs `mise run docs-api` up) -# mcp/… mcp loop (rebuild the local server) -# evals/submodules/agent-skills/… skills loop (no re-sync; read live via symlink) +# supabase/apps/docs/content/… docs loop (re-embed via docs-index; needs `mise run docs-api` up) +# submodules/mcp/… mcp loop (rebuild the local server) +# submodules/agent-skills/… skills loop (no re-sync; read live via symlink) # # The enabler patches live as [eval-workspace-*] commits below your work (see # apply-patches.sh), so your unstaged edit is the ONLY working diff — stashing @@ -20,9 +20,9 @@ # AB_DRYRUN=1 prints the resolved plan and exits (no runs, no side effects). # AB_EVAL_CMD / AB_SYNC_CMD override the eval / sync step (testing; see ab.test.sh). set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." -[ $# -ge 3 ] || { echo "usage: scripts/ab.sh [more-paths...]" >&2; exit 2; } +[ $# -ge 3 ] || { echo "usage: workspace/scripts/ab.sh [more-paths...]" >&2; exit 2; } EVAL="$1"; EXP="$2"; shift 2 PATHS=("$@") @@ -34,11 +34,11 @@ done # --- clone + loop from the first path --- case "${PATHS[0]}" in - supabase/apps/docs/content/*) LOOP=docs; CLONE=supabase; STRIP=supabase/; PREFIX=supabase/apps/docs/content/ ;; - supabase/apps/docs/*) echo "docs A/B works on content pages (supabase/apps/docs/content/…) — other docs files aren't part of the embed loop: ${PATHS[0]}" >&2; exit 2 ;; - mcp/*) LOOP=mcp; CLONE=mcp; STRIP=mcp/; PREFIX=mcp/ ;; - evals/submodules/agent-skills/*) LOOP=skills; CLONE=evals/submodules/agent-skills; STRIP=evals/submodules/agent-skills/; PREFIX=evals/submodules/agent-skills/ ;; - *) echo "path must be under supabase/apps/docs/content/, mcp/, or evals/submodules/agent-skills/: ${PATHS[0]}" >&2; exit 2 ;; + supabase/apps/docs/content/*) LOOP=docs; CLONE=supabase; STRIP=supabase/; PREFIX=supabase/apps/docs/content/ ;; + supabase/apps/docs/*) echo "docs A/B works on content pages (supabase/apps/docs/content/…) — other docs files aren't part of the embed loop: ${PATHS[0]}" >&2; exit 2 ;; + submodules/mcp/*) LOOP=mcp; CLONE=submodules/mcp; STRIP=submodules/mcp/; PREFIX=submodules/mcp/ ;; + submodules/agent-skills/*) LOOP=skills; CLONE=submodules/agent-skills; STRIP=submodules/agent-skills/; PREFIX=submodules/agent-skills/ ;; + *) echo "path must be under supabase/apps/docs/content/, submodules/mcp/, or submodules/agent-skills/: ${PATHS[0]}" >&2; exit 2 ;; esac # every path must live in this loop's editable scope; build clone-relative paths @@ -50,7 +50,7 @@ for p in "${PATHS[@]}"; do esac done -MCP="$PWD/mcp/packages/mcp-server-supabase" +MCP="$PWD/submodules/mcp/packages/mcp-server-supabase" CONTENT_URL="http://127.0.0.1:3001/docs/api/graphql" RUN_ENV=() case "$LOOP" in @@ -61,8 +61,8 @@ esac sync() { if [ -n "${AB_SYNC_CMD:-}" ]; then bash -c "$AB_SYNC_CMD"; return; fi # override/test hook case "$LOOP" in - docs) scripts/docs-index.sh ;; - mcp) ( cd mcp && pnpm build ) ;; + docs) workspace/scripts/docs-index.sh ;; + mcp) ( cd submodules/mcp && pnpm build ) ;; skills) : ;; esac } @@ -92,10 +92,10 @@ done if [ "$LOOP" = docs ]; then : "${OPENAI_API_KEY:?OPENAI_API_KEY not in keychain — the docs re-embed needs it}" curl -sf -o /dev/null "$CONTENT_URL" || { echo "docs-api not reachable on :3001 — run \`mise run docs-api\` in another terminal first" >&2; exit 1; } - [ -d "$MCP/dist" ] || { echo "building local mcp (needed for search_docs routing)…"; ( cd mcp && pnpm install && pnpm build ); } + [ -d "$MCP/dist" ] || { echo "building local mcp (needed for search_docs routing)…"; ( cd submodules/mcp && pnpm install && pnpm build ); } fi -RES="evals/results/$EXP/$EVAL.json" +RES="results/$EXP/$EVAL.json" OUT="results-ab"; mkdir -p "$OUT" run_eval() { # $1 = label @@ -103,13 +103,13 @@ run_eval() { # $1 = label if [ -n "${AB_EVAL_CMD:-}" ]; then RES="$RES" AB_LABEL="$1" bash -c "$AB_EVAL_CMD" # override/test hook; must write $RES else - ( cd evals && env ${RUN_ENV[@]+"${RUN_ENV[@]}"} pnpm eval --eval "$EVAL" --experiment "$EXP" --runs 1 ) + env ${RUN_ENV[@]+"${RUN_ENV[@]}"} pnpm eval --eval "$EVAL" --experiment "$EXP" --runs 1 fi [ -f "$RES" ] || { echo "no result at $RES — check the eval/experiment ids" >&2; exit 1; } cp "$RES" "$OUT/$EVAL.$1.json" # per-arm receipt: capture provenance AFTER this arm's sync, so baseline and # treatment records differ by exactly the edit under test (report-only) - node scripts/provenance.mjs --embed "$OUT/$EVAL.$1.json" + node workspace/scripts/provenance.mjs --embed "$OUT/$EVAL.$1.json" } # Restoration is ONE idempotent path: pop the stash AND re-sync, so the index/ @@ -126,7 +126,7 @@ restore() { return 1 fi if ! sync; then - echo "ERROR: post-restore re-sync failed — the index/build does not reflect your edit (docs: scripts/docs-index.sh; mcp: pnpm -C mcp build)" >&2 + echo "ERROR: post-restore re-sync failed — the index/build does not reflect your edit (docs: workspace/scripts/docs-index.sh; mcp: pnpm -C submodules/mcp build)" >&2 return 1 fi } diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh index a9c40761..ef22c2ec 100755 --- a/workspace/scripts/ab.test.sh +++ b/workspace/scripts/ab.test.sh @@ -1,18 +1,19 @@ #!/usr/bin/env bash -# Zero-cost integration test for scripts/ab.sh. Fakes the eval via AB_EVAL_CMD -# (no model, no OpenAI, no services) and proves: the scoped stash/restore + result -# copy + comparison work, AND the selected edit and index are byte-identical after -# the run — including when a run fails mid-A/B (trap must restore). +# Zero-cost integration test for workspace/scripts/ab.sh. Fakes the eval via +# AB_EVAL_CMD (no model, no OpenAI, no services) and proves: the scoped +# stash/restore + result copy + comparison work, AND the selected edit and +# index are byte-identical after the run — including when a run fails mid-A/B +# (trap must restore). # # SAFETY: only ever touches a tracked skill file that is currently CLEAN (no # working-tree or staged changes); it never checks out a file with your edits. # If no clean skill file exists it SKIPS. Skills loop → sync is a no-op, so -# nothing else in the workspace is affected. Run: bash scripts/ab.test.sh +# nothing else in the workspace is affected. Run: bash workspace/scripts/ab.test.sh set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." -SK=evals/submodules/agent-skills -[ -e "$SK/.git" ] || { echo "SKIP: agent-skills not cloned (run: mise run setup)"; exit 0; } +SK=submodules/agent-skills +[ -e "$SK/.git" ] || { echo "SKIP: agent-skills submodule not initialized (run: mise run setup)"; exit 0; } # pick a tracked SKILL.md with NO local changes, so restoring it can't lose work REL="" @@ -22,13 +23,14 @@ done [ -n "$REL" ] || { echo "SKIP: no CLEAN tracked skill file to test with"; exit 0; } FILE="$SK/$REL" EVAL=ab-selftest; EXP=claude-sonnet-5 -RES="evals/results/$EXP/$EVAL.json" +RES="results/$EXP/$EVAL.json" # only now (after selecting a clean file) arm cleanup: revert our own edit, drop fixtures +MCP=submodules/mcp MCPREL=packages/mcp-server-supabase/src/transports/stdio.ts cleanup() { git -C "$SK" checkout -q -- "$REL" 2>/dev/null || true - [ -e mcp/.git ] && git -C mcp checkout -q -- "$MCPREL" 2>/dev/null || true + [ -e "$MCP/.git" ] && git -C "$MCP" checkout -q -- "$MCPREL" 2>/dev/null || true rm -f "$RES" "results-ab/$EVAL".*.json } trap cleanup EXIT @@ -50,7 +52,7 @@ SYNC="echo x >> $CNT" # --- happy path --- : > "$CNT" -ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest.out 2>&1 \ +ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest.out 2>&1 \ || { echo "FAIL: ab exited nonzero (happy path)"; sed 's/^/ /' /tmp/ab_selftest.out; fail=$((fail+1)); } ck "edit preserved after success" "$(git -C "$SK" hash-object "$REL")" "$edited" ck "index untouched after success" "$(git -C "$SK" diff --cached -- "$REL")" "$index0" @@ -62,29 +64,29 @@ ck "sync ran 3x (incl. post-restore)" "$(wc -l < "$CNT" | tr -d ' ')" "3" # --- failure path: baseline dies AFTER stash; ab's trap must restore the edit AND re-sync --- : > "$CNT" -AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest2.out 2>&1 || true +AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest2.out 2>&1 || true ck "edit restored after mid-run failure" "$(git -C "$SK" hash-object "$REL")" "$edited" ck "index untouched after failure" "$(git -C "$SK" diff --cached -- "$REL")" "$index0" ck "trap re-synced after failure (3x)" "$(wc -l < "$CNT" | tr -d ' ')" "3" -ck "failure exit status preserved" "$(AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/dev/null 2>&1; echo $?)" "7" +ck "failure exit status preserved" "$(AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/dev/null 2>&1; echo $?)" "7" # --- restore-sync fails (3rd sync): must fail the run, keep the edit, skip the report --- : > "$CNT" SYNC_FAIL3="echo x >> $CNT; [ \$(wc -l < $CNT) -lt 3 ] || { echo sync3-boom >&2; exit 9; }" -rc3=$(ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC_FAIL3" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest3.out 2>&1; echo $?) +rc3=$(ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC_FAIL3" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest3.out 2>&1; echo $?) ck "restore-sync failure fails the run" "$([ "$rc3" -ne 0 ] && echo nonzero || echo zero)" "nonzero" ck "edit intact after restore-sync failure" "$(git -C "$SK" hash-object "$REL")" "$edited" ck "reports re-sync error" "$(grep -c 'post-restore re-sync failed' /tmp/ab_selftest3.out)" "1" ck "no A/B report on failed restore" "$(grep -c '=== A/B' /tmp/ab_selftest3.out)" "0" # --- docs loop must reject a second path outside the content scope --- -rc4=$(AB_DRYRUN=1 bash scripts/ab.sh e x supabase/apps/docs/content/a.mdx supabase/apps/studio/foo.ts >/dev/null 2>&1; echo $?) +rc4=$(AB_DRYRUN=1 bash workspace/scripts/ab.sh e x supabase/apps/docs/content/a.mdx supabase/apps/studio/foo.ts >/dev/null 2>&1; echo $?) ck "rejects non-content path in docs loop" "$rc4" "2" # --- dirty clone index (e.g. intent-to-add residue) must be refused before stashing --- echo probe > "$SK/.ab-selftest-idx-probe" git -C "$SK" add -N .ab-selftest-idx-probe -rc5=$(ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest5.out 2>&1; echo $?) +rc5=$(ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" >/tmp/ab_selftest5.out 2>&1; echo $?) git -C "$SK" reset -q -- .ab-selftest-idx-probe && rm -f "$SK/.ab-selftest-idx-probe" ck "refuses dirty clone index" "$rc5" "1" ck "explains staged-index refusal" "$(grep -c 'staged changes' /tmp/ab_selftest5.out)" "1" @@ -92,20 +94,20 @@ ck "explains staged-index refusal" "$(grep -c 'staged changes' /tmp/ab_selftes # --- patch-owned file A/B (the commit model's core win): an unstaged edit on a # file the mcp enabler patch owns must A/B cleanly, and the plumbing marker # commit must be untouched on both success and mid-run failure --- -if [ -e mcp/.git ] && git -C mcp diff --quiet -- "$MCPREL" && git -C mcp diff --cached --quiet --ita-visible-in-index; then - marker0=$(git -C mcp rev-parse HEAD) - printf '\n// ab-selftest edit\n' >> "mcp/$MCPREL" - edited2=$(git -C mcp hash-object "$MCPREL") - ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "mcp/$MCPREL" >/tmp/ab_selftest6.out 2>&1 \ +if [ -e "$MCP/.git" ] && git -C "$MCP" diff --quiet -- "$MCPREL" && git -C "$MCP" diff --cached --quiet --ita-visible-in-index; then + marker0=$(git -C "$MCP" rev-parse HEAD) + printf '\n// ab-selftest edit\n' >> "$MCP/$MCPREL" + edited2=$(git -C "$MCP" hash-object "$MCPREL") + ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$MCP/$MCPREL" >/tmp/ab_selftest6.out 2>&1 \ || { echo "FAIL: patch-owned A/B exited nonzero"; sed 's/^/ /' /tmp/ab_selftest6.out; fail=$((fail+1)); } - ck "patch-owned: edit preserved" "$(git -C mcp hash-object "$MCPREL")" "$edited2" - ck "patch-owned: marker commit intact" "$(git -C mcp rev-parse HEAD)" "$marker0" - AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash scripts/ab.sh "$EVAL" "$EXP" "mcp/$MCPREL" >/dev/null 2>&1 || true - ck "patch-owned: edit restored after failure" "$(git -C mcp hash-object "$MCPREL")" "$edited2" - ck "patch-owned: marker intact after failure" "$(git -C mcp rev-parse HEAD)" "$marker0" - git -C mcp checkout -q -- "$MCPREL" + ck "patch-owned: edit preserved" "$(git -C "$MCP" hash-object "$MCPREL")" "$edited2" + ck "patch-owned: marker commit intact" "$(git -C "$MCP" rev-parse HEAD)" "$marker0" + AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$MCP/$MCPREL" >/dev/null 2>&1 || true + ck "patch-owned: edit restored after failure" "$(git -C "$MCP" hash-object "$MCPREL")" "$edited2" + ck "patch-owned: marker intact after failure" "$(git -C "$MCP" rev-parse HEAD)" "$marker0" + git -C "$MCP" checkout -q -- "$MCPREL" else - echo "SKIP: mcp not cloned/clean — patch-owned A/B regression not run" + echo "SKIP: mcp not initialized/clean — patch-owned A/B regression not run" fi echo "ab.test: $pass passed, $fail failed" diff --git a/workspace/scripts/affected-task.sh b/workspace/scripts/affected-task.sh index 02c0782f..ffda3d53 100755 --- a/workspace/scripts/affected-task.sh +++ b/workspace/scripts/affected-task.sh @@ -2,7 +2,7 @@ # Which evals does a change affect? Maps changed skill/docs/mcp paths to a # ready-to-run eval command. e.g. mise run affected apps/docs/content/guides/auth/x.mdx set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." ROOT="$PWD" -cd evals/apps/framework && exec node --import tsx/esm "$ROOT/scripts/affected.ts" "$@" +cd apps/framework && exec node --import tsx/esm "$ROOT/workspace/scripts/affected.ts" "$@" diff --git a/workspace/scripts/affected.ts b/workspace/scripts/affected.ts index 7683e21d..ae6d916b 100755 --- a/workspace/scripts/affected.ts +++ b/workspace/scripts/affected.ts @@ -1,16 +1,11 @@ #!/usr/bin/env tsx -const { - existsSync, - readFileSync, - readdirSync, - statSync, -} = require("node:fs"); -const { createRequire } = require("node:module"); -const { basename, dirname, join } = require("node:path"); -const { pathToFileURL } = require("node:url"); - -const EVALS_ROOT = join(dirname(process.argv[1]), "..", "evals"); -const requireFromFramework = createRequire(join(EVALS_ROOT, "apps/framework/package.json")); +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { createRequire } from "node:module"; +import { basename, dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const ROOT = join(dirname(process.argv[1]), "..", ".."); +const requireFromFramework = createRequire(join(ROOT, "apps/framework/package.json")); const ALIASES: Record = { functions: "edge-functions", rest: "data-api", @@ -26,7 +21,7 @@ type Eval = { }; async function loadExperiments(): Promise { - const dir = join(EVALS_ROOT, "experiments"); + const dir = join(ROOT, "experiments"); const experiments: Experiment[] = []; for (const file of readdirSync(dir).filter((file) => file.endsWith(".ts")).sort()) { @@ -43,10 +38,10 @@ async function loadExperiments(): Promise { } async function discoverEvals(): Promise { - // Resolve from evals' workspace because this script intentionally lives outside that package. + // Resolve from the repo root because this script intentionally lives outside that package. const parserPath = requireFromFramework.resolve("@supabase-evals/core/eval-markdown"); const { parseEvalMarkdown } = await import(pathToFileURL(parserPath).href); - const dir = join(EVALS_ROOT, "evals"); + const dir = join(ROOT, "evals"); if (!existsSync(dir)) return []; const evals: Eval[] = []; diff --git a/workspace/scripts/apply-patches.sh b/workspace/scripts/apply-patches.sh index 6c724acc..85f7a539 100755 --- a/workspace/scripts/apply-patches.sh +++ b/workspace/scripts/apply-patches.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Materialize the enabler patches as identifiable LOCAL COMMITS in each clone: +# Materialize the enabler patches as identifiable LOCAL COMMITS in the +# supabase clone and the mcp submodule working tree: # [eval-workspace-local] dev shim — must never leave this machine # [eval-workspace-upstream] upstream candidate — leaves ONLY via # `mise run publish … --with ` (reworded) @@ -8,68 +9,70 @@ # # Idempotent: a patch whose marker commit already exists is skipped; patch # content found uncommitted in the working tree (the old model) is migrated -# into a commit. Also installs a pre-push guard in every publishable repo -# (incl. the agent-skills submodule) that blocks marker commits from being -# pushed anywhere. +# into a commit. Also installs a pre-push guard in the supabase clone and +# the mcp submodule (the two publishable repos) that blocks marker commits +# from being pushed anywhere. A repo whose dir is absent (supabase not +# cloned) or whose submodule is uninitialized (empty working tree, no .git) +# is skipped cleanly — never install anything on the host repo. set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." ROOT="$PWD" -source scripts/patches-lib.sh +source workspace/scripts/patches-lib.sh apply_one() { - local repo="$1" patch="$2" subject sha h s + local dir="$1" patch="$2" subject sha h s subject=$(patch_subject "$patch") # marker commit already present? (search ALL local-only history) sha="" while read -r h s; do [ "$s" = "$subject" ] && { sha="$h"; break; } done </dev/null) +$(git -C "$dir" log --format='%H %s' HEAD --not --remotes 2>/dev/null) EOF if [ -n "$sha" ]; then # the .patch file is canonical — the commit must still match it EXACTLY: # rebuild parent-tree + patch in a temp index and compare tree OIDs local tmpidx want got tmpidx=$(mktemp /tmp/eval-workspace-idx-XXXXXX); rm -f "$tmpidx" # keep only the name: git must create the index itself - want=$(git -C "$repo" rev-parse "$sha^{tree}") + want=$(git -C "$dir" rev-parse "$sha^{tree}") got="" - if GIT_INDEX_FILE="$tmpidx" git -C "$repo" read-tree "$sha^" 2>/dev/null \ - && GIT_INDEX_FILE="$tmpidx" git -C "$repo" apply --cached "$ROOT/$patch" 2>/dev/null; then - got=$(GIT_INDEX_FILE="$tmpidx" git -C "$repo" write-tree 2>/dev/null || echo "") + if GIT_INDEX_FILE="$tmpidx" git -C "$dir" read-tree "$sha^" 2>/dev/null \ + && GIT_INDEX_FILE="$tmpidx" git -C "$dir" apply --cached "$ROOT/$patch" 2>/dev/null; then + got=$(GIT_INDEX_FILE="$tmpidx" git -C "$dir" write-tree 2>/dev/null || echo "") fi rm -f "$tmpidx" if [ "$got" = "$want" ]; then - echo " $(basename "$patch") already committed in $repo" + echo " $(basename "$patch") already committed in $dir" else - echo " ERROR: the $(basename "$patch") commit in $repo differs from the canonical patch file." >&2 - echo " refresh the file from the commit: git -C $repo diff $sha^ $sha > $patch" >&2 - echo " or drop the commit and re-apply: git -C $repo rebase --onto $sha^ $sha && mise run apply-patches" >&2 + echo " ERROR: the $(basename "$patch") commit in $dir differs from the canonical patch file." >&2 + echo " refresh the file from the commit: git -C $dir diff $sha^ $sha > $patch" >&2 + echo " or drop the commit and re-apply: git -C $dir rebase --onto $sha^ $sha && mise run apply-patches" >&2 exit 1 fi return fi # Build the commit from the CANONICAL patch via the index — never stage whole # files, so user edits sitting in patch-owned files can't be absorbed. - if git -C "$repo" apply --index --check "$ROOT/$patch" 2>/dev/null; then - git -C "$repo" apply --index "$ROOT/$patch" # fresh clone: index + worktree - elif git -C "$repo" apply --cached --check "$ROOT/$patch" 2>/dev/null; then - git -C "$repo" apply --cached "$ROOT/$patch" # content already in worktree; extra edits stay unstaged above + if git -C "$dir" apply --index --check "$ROOT/$patch" 2>/dev/null; then + git -C "$dir" apply --index "$ROOT/$patch" # fresh clone: index + worktree + elif git -C "$dir" apply --cached --check "$ROOT/$patch" 2>/dev/null; then + git -C "$dir" apply --cached "$ROOT/$patch" # content already in worktree; extra edits stay unstaged above else - echo " ERROR: $(basename "$patch") does not apply cleanly to $repo — regenerate it (patches/README.md)" >&2 + echo " ERROR: $(basename "$patch") does not apply cleanly to $dir — regenerate it (workspace/patches/README.md)" >&2 exit 1 fi # the staged diff must be exactly the canonical patch (payload comparison) - if ! diff -q <(git -C "$repo" diff --cached | grep -E '^[+-]' | grep -vE '^(\+\+\+|---)') \ + if ! diff -q <(git -C "$dir" diff --cached | grep -E '^[+-]' | grep -vE '^(\+\+\+|---)') \ <(grep -E '^[+-]' "$ROOT/$patch" | grep -vE '^(\+\+\+|---)') >/dev/null 2>&1; then - git -C "$repo" reset -q + git -C "$dir" reset -q echo " ERROR: staged diff for $(basename "$patch") differs from the canonical patch — aborted (index reset)" >&2 exit 1 fi GIT_AUTHOR_NAME=eval-workspace GIT_AUTHOR_EMAIL=eval-workspace@local \ GIT_COMMITTER_NAME=eval-workspace GIT_COMMITTER_EMAIL=eval-workspace@local \ - git -C "$repo" commit -q -m "$subject" \ - -m "eval-workspace plumbing ($(patch_kind "$patch") kind), generated from $patch. Do not push; see patches/README.md." - echo " committed $(basename "$patch") -> $repo" + git -C "$dir" commit -q -m "$subject" \ + -m "eval-workspace plumbing ($(patch_kind "$patch") kind), generated from $patch. Do not push; see workspace/patches/README.md." + echo " committed $(basename "$patch") -> $dir" } install_pre_push_guard() { @@ -133,10 +136,11 @@ HOOK echo "Applying enabler patches (as local marker commits):" for repo in $PATCH_REPOS; do - if [ ! -e "$repo/.git" ]; then echo " $repo not cloned — skipping"; continue; fi - git -C "$repo" diff --cached --quiet --ita-visible-in-index \ - || { echo " ERROR: $repo index has staged changes — unstage first (git -C $repo status)" >&2; exit 1; } - for p in $(patches_for "$repo"); do apply_one "$repo" "$p"; done + dir=$(repo_dir "$repo") + if [ ! -e "$dir/.git" ]; then echo " $repo ($dir) not present — skipping"; continue; fi + git -C "$dir" diff --cached --quiet --ita-visible-in-index \ + || { echo " ERROR: $dir index has staged changes — unstage first (git -C $dir status)" >&2; exit 1; } + for p in $(patches_for "$repo"); do apply_one "$dir" "$p"; done done for name in $PUBLISH_REPOS; do dir=$(repo_dir "$name") diff --git a/workspace/scripts/clone-docs.sh b/workspace/scripts/clone-docs.sh index e56ee7bd..70b2f6fd 100755 --- a/workspace/scripts/clone-docs.sh +++ b/workspace/scripts/clone-docs.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -cd "$(dirname "$0")/.." -source scripts/patches-lib.sh +cd "$(dirname "$0")/../.." +source workspace/scripts/patches-lib.sh SUPABASE_REMOTE="${SUPABASE_REMOTE:-$(repo_remote supabase)}" if [ -e supabase/.git ]; then diff --git a/workspace/scripts/docs-api.sh b/workspace/scripts/docs-api.sh index 671f03f4..aa1e82c1 100755 --- a/workspace/scripts/docs-api.sh +++ b/workspace/scripts/docs-api.sh @@ -2,9 +2,9 @@ # Serve the docs content GraphQL API locally (standalone adapter) for search_docs. # Point evals at it: SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." -source scripts/load-keys.sh +source workspace/scripts/load-keys.sh set -a source supabase/apps/docs/.env.development set +a @@ -17,4 +17,4 @@ OPENAI_API_KEY="$OPENAI_API_KEY" \ exec pnpm --dir supabase/apps/docs exec tsx \ --conditions=react-server \ --tsconfig tsconfig.json \ - ../../../scripts/docs-content-api.ts + ../../../workspace/scripts/docs-content-api.ts diff --git a/workspace/scripts/docs-content-api.ts b/workspace/scripts/docs-content-api.ts index 74dadd83..e869d938 100644 --- a/workspace/scripts/docs-content-api.ts +++ b/workspace/scripts/docs-content-api.ts @@ -1,5 +1,5 @@ import { createServer } from 'node:http' -import { GET, OPTIONS, POST } from '../supabase/apps/docs/app/api/graphql/route.ts' +import { GET, OPTIONS, POST } from '../../supabase/apps/docs/app/api/graphql/route.ts' const handlers = { GET, OPTIONS, POST } const port = Number(process.env.PORT ?? 3001) diff --git a/workspace/scripts/docs-down.sh b/workspace/scripts/docs-down.sh index cbf7c75f..182dd86b 100755 --- a/workspace/scripts/docs-down.sh +++ b/workspace/scripts/docs-down.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." supabase stop --workdir supabase diff --git a/workspace/scripts/docs-index.sh b/workspace/scripts/docs-index.sh index 1646a208..fc68d07a 100755 --- a/workspace/scripts/docs-index.sh +++ b/workspace/scripts/docs-index.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash set -euo pipefail -cd "$(dirname "$0")/.." -if ! git -C supabase apply --reverse --check "$PWD/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then - echo 'ERROR: fail-closed index patch is not applied; run scripts/apply-patches.sh' >&2 +cd "$(dirname "$0")/../.." +if ! git -C supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then + echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 exit 1 fi -source scripts/load-keys.sh +source workspace/scripts/load-keys.sh set -a source supabase/apps/docs/.env.development set +a @@ -16,10 +16,10 @@ export NEXT_PUBLIC_SUPABASE_URL="$API_URL" export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" export SUPABASE_SECRET_KEY="$SECRET_KEY" : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" -scripts/openai-preflight.sh # fail fast if the key can't run embeddings +workspace/scripts/openai-preflight.sh # fail fast if the key can't run embeddings # Deliberately allow embedding without prod-only sources (e.g. lint-warnings, # which needs DOCS_GITHUB_APP_*). Sources gate their own skip on this flag. export DOCS_EMBED_ALLOW_MISSING_SOURCES=1 pnpm --dir supabase/apps/docs run embeddings -node scripts/provenance.mjs --stamp-docs-index +node workspace/scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-seed.sh b/workspace/scripts/docs-seed.sh index 32809785..60aca009 100755 --- a/workspace/scripts/docs-seed.sh +++ b/workspace/scripts/docs-seed.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash set -euo pipefail -cd "$(dirname "$0")/.." -if ! git -C supabase apply --reverse --check "$PWD/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then - echo 'ERROR: fail-closed index patch is not applied; run scripts/apply-patches.sh' >&2 +cd "$(dirname "$0")/../.." +if ! git -C supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then + echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 exit 1 fi -source scripts/load-keys.sh +source workspace/scripts/load-keys.sh set -a source supabase/apps/docs/.env.development set +a @@ -16,7 +16,7 @@ export NEXT_PUBLIC_SUPABASE_URL="$API_URL" export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" export SUPABASE_SECRET_KEY="$SECRET_KEY" : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" -scripts/openai-preflight.sh # fail fast if the key can't run embeddings +workspace/scripts/openai-preflight.sh # fail fast if the key can't run embeddings # Deliberately allow embedding without prod-only sources (e.g. lint-warnings, # which needs DOCS_GITHUB_APP_*). Sources gate their own skip on this flag. export DOCS_EMBED_ALLOW_MISSING_SOURCES=1 @@ -32,4 +32,4 @@ if [ "$confirmation" != seed ]; then fi pnpm --dir supabase/apps/docs run embeddings:refresh -node scripts/provenance.mjs --stamp-docs-index +node workspace/scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-up.sh b/workspace/scripts/docs-up.sh index af026a0f..1971d65f 100755 --- a/workspace/scripts/docs-up.sh +++ b/workspace/scripts/docs-up.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." supabase start --workdir supabase \ -x realtime,storage-api,imgproxy,mailpit,postgres-meta,studio,edge-runtime,logflare,vector,supavisor diff --git a/workspace/scripts/eval.sh b/workspace/scripts/eval.sh index f9b406d3..6f2cef7a 100755 --- a/workspace/scripts/eval.sh +++ b/workspace/scripts/eval.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Run evals with the workspace env applied. All args pass through to `pnpm eval`. -# API keys come from the macOS keychain if present (see README), else evals' .env. -# e.g. scripts/eval.sh --eval investigate-auth-001 --experiment claude-code-sonnet-5 +# API keys come from the macOS keychain if present (see README), else the repo-root .env. +# e.g. workspace/scripts/eval.sh --eval investigate-auth-001 --experiment claude-code-sonnet-5 set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." -[ -e evals/.env ] || { echo "evals/.env missing — run: mise run setup" >&2; exit 1; } -source scripts/load-keys.sh -cd evals && pnpm eval "$@" +[ -e .env ] || { echo ".env missing — run: mise run setup" >&2; exit 1; } +source workspace/scripts/load-keys.sh +pnpm eval "$@" diff --git a/workspace/scripts/hooks.test.sh b/workspace/scripts/hooks.test.sh index e4ad6445..d1fe81ef 100755 --- a/workspace/scripts/hooks.test.sh +++ b/workspace/scripts/hooks.test.sh @@ -1,44 +1,59 @@ #!/usr/bin/env bash # Self-test for the pre-push guard lifecycle (install / chain / reinstall). -# Uses the mcp clone's hooks dir — .git internals only, repo content untouched; -# every fixture is removed and the real guard is reinstalled at the end. +# Exercises the guard in every publishable repo present locally — the +# supabase clone and the mcp submodule working tree. .git internals only, +# repo content untouched; every fixture is removed and the real guard +# reinstalled at the end. A repo that isn't cloned/initialized, or that +# already has a real chained hook, is skipped for that repo only. set -euo pipefail -cd "$(dirname "$0")/.." - -[ -e mcp/.git ] || { echo "SKIP: mcp not cloned"; exit 0; } -HOOKS=$(git -C mcp rev-parse --path-format=absolute --git-path hooks) -HOOK="$HOOKS/pre-push" -CHAINED="$HOOKS/pre-push.eval-workspace-chained" -[ ! -e "$CHAINED" ] || { echo "SKIP: $CHAINED already exists (real chained hook — not touching it)"; exit 0; } +cd "$(dirname "$0")/../.." +source workspace/scripts/patches-lib.sh pass=0; fail=0 ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $1 (want[$3] got[$2])"; fi; } -cleanup() { rm -f "$CHAINED"; scripts/apply-patches.sh >/dev/null 2>&1 || true; } + +CHAINED_PATHS=() +cleanup() { for c in ${CHAINED_PATHS[@]+"${CHAINED_PATHS[@]}"}; do rm -f "$c"; done; workspace/scripts/apply-patches.sh >/dev/null 2>&1 || true; } trap cleanup EXIT -# baseline: generated guard present -scripts/apply-patches.sh >/dev/null -ck "guard installed" "$(grep -c 'eval-workspace pre-push guard' "$HOOK")" "1" - -# foreign hook appears -> next install chains it -printf '#!/usr/bin/env bash\n# foreign hook one\nexit 0\n' > "$HOOK"; chmod +x "$HOOK" -scripts/apply-patches.sh >/dev/null 2>&1 -ck "foreign hook chained" "$(grep -c 'foreign hook one' "$CHAINED" 2>/dev/null || echo 0)" "1" -ck "guard reinstalled" "$(grep -c 'eval-workspace pre-push guard' "$HOOK")" "1" - -# a DIFFERENT foreign hook overwrites the generated guard -> reinstall must -# refuse rather than silently clobber the previously chained hook -printf '#!/usr/bin/env bash\n# foreign hook two\nexit 0\n' > "$HOOK"; chmod +x "$HOOK" -rc=$(scripts/apply-patches.sh >/dev/null 2>&1; echo $?) -ck "reinstall refuses to clobber chained" "$rc" "1" -ck "chained hook one preserved" "$(grep -c 'foreign hook one' "$CHAINED")" "1" -ck "foreign hook two preserved" "$(grep -c 'foreign hook two' "$HOOK")" "1" - -# identical foreign hook (e.g. re-copied) -> harmless, install proceeds -cp "$CHAINED" "$HOOK" -rc=$(scripts/apply-patches.sh >/dev/null 2>&1; echo $?) -ck "identical foreign re-chain ok" "$rc" "0" -ck "guard active again" "$(grep -c 'eval-workspace pre-push guard' "$HOOK")" "1" +test_guard_in() { + local repo="$1" dir hooks hook chained rc + dir=$(repo_dir "$repo") + [ -e "$dir/.git" ] || { echo "SKIP $repo: $dir not present"; return 0; } + hooks=$(git -C "$dir" rev-parse --path-format=absolute --git-path hooks) + hook="$hooks/pre-push" + chained="$hooks/pre-push.eval-workspace-chained" + [ ! -e "$chained" ] || { echo "SKIP $repo: $chained already exists (real chained hook — not touching it)"; return 0; } + CHAINED_PATHS+=("$chained") + + # baseline: generated guard present + workspace/scripts/apply-patches.sh >/dev/null + ck "$repo: guard installed" "$(grep -c 'eval-workspace pre-push guard' "$hook")" "1" + + # foreign hook appears -> next install chains it + printf '#!/usr/bin/env bash\n# foreign hook one\nexit 0\n' > "$hook"; chmod +x "$hook" + workspace/scripts/apply-patches.sh >/dev/null 2>&1 + ck "$repo: foreign hook chained" "$(grep -c 'foreign hook one' "$chained" 2>/dev/null || echo 0)" "1" + ck "$repo: guard reinstalled" "$(grep -c 'eval-workspace pre-push guard' "$hook")" "1" + + # a DIFFERENT foreign hook overwrites the generated guard -> reinstall must + # refuse rather than silently clobber the previously chained hook + printf '#!/usr/bin/env bash\n# foreign hook two\nexit 0\n' > "$hook"; chmod +x "$hook" + rc=$(workspace/scripts/apply-patches.sh >/dev/null 2>&1; echo $?) + ck "$repo: reinstall refuses to clobber chained" "$rc" "1" + ck "$repo: chained hook one preserved" "$(grep -c 'foreign hook one' "$chained")" "1" + ck "$repo: foreign hook two preserved" "$(grep -c 'foreign hook two' "$hook")" "1" + + # identical foreign hook (e.g. re-copied) -> harmless, install proceeds + cp "$chained" "$hook" + rc=$(workspace/scripts/apply-patches.sh >/dev/null 2>&1; echo $?) + ck "$repo: identical foreign re-chain ok" "$rc" "0" + ck "$repo: guard active again" "$(grep -c 'eval-workspace pre-push guard' "$hook")" "1" +} + +for r in $PUBLISH_REPOS; do + test_guard_in "$r" +done echo "hooks.test: $pass passed, $fail failed" [ "$fail" -eq 0 ] diff --git a/workspace/scripts/manifest.mjs b/workspace/scripts/manifest.mjs index f3184e3e..142965f5 100755 --- a/workspace/scripts/manifest.mjs +++ b/workspace/scripts/manifest.mjs @@ -1,9 +1,15 @@ #!/usr/bin/env node -// Loader + CLI for manifest.json — the single source of truth for the repos -// this workspace wires together: checkout dirs, upstream remotes, and the -// ordered enabler-patch set per repo (patch names without the patches/ -// prefix or .patch suffix; "localPatches" marks the dev shims that must never -// reach an upstream PR — see patches/README.md for per-patch provenance). +// Loader + CLI for manifest.json (lives at workspace/manifest.json, beside +// scripts/) — the single source of truth for the repos this workspace wires +// together: repo-root-relative checkout dirs, upstream remotes (clone +// entries only), and the ordered enabler-patch set per repo (patch names +// without the patches/ prefix or .patch suffix; "localPatches" marks the dev +// shims that must never reach an upstream PR — see patches/README.md). +// `kind: "submodule"` repos (mcp, skills) have no remote — .gitmodules owns +// that — and are never cloned by our scripts; a repo without `kind` is a +// plain clone and MUST have a remote. Every `dir` is repo-root-relative; +// resolving it is the caller's job (bash scripts `cd` to the repo root, +// two levels up from workspace/scripts, before using it). // // loadManifest() is THE loader: every consumer (the CLI below, the bash // facade scripts/patches-lib.sh through it, and provenance.mjs via import) @@ -38,7 +44,14 @@ export function loadManifest() { } for (const [name, repo] of Object.entries(manifest.repos)) { if (!isStr(repo?.dir)) throw new Error(`repos.${name}.dir must be a non-empty string`); - if (repo.remote !== undefined && !isStr(repo.remote)) throw new Error(`repos.${name}.remote must be a non-empty string`); + if (repo.kind !== undefined && repo.kind !== "submodule") { + throw new Error(`repos.${name}.kind must be "submodule" when present`); + } + if (repo.kind === "submodule") { + if (repo.remote !== undefined) throw new Error(`repos.${name}.remote must be absent for kind:"submodule" (.gitmodules owns the remote)`); + } else if (!isStr(repo.remote)) { + throw new Error(`repos.${name}.remote must be a non-empty string (required for clone entries)`); + } if (repo.patches !== undefined && !isStrArr(repo.patches)) throw new Error(`repos.${name}.patches must be a non-empty array of strings`); if (repo.localPatches !== undefined) { if (!isStrArr(repo.localPatches)) throw new Error(`repos.${name}.localPatches must be a non-empty array of strings`); diff --git a/workspace/scripts/mcp-eval.sh b/workspace/scripts/mcp-eval.sh index 16c8b911..98051f79 100755 --- a/workspace/scripts/mcp-eval.sh +++ b/workspace/scripts/mcp-eval.sh @@ -2,9 +2,6 @@ # Run evals against the local mcp build (mise task `mcp-eval` builds it first). # Add SUPABASE_CONTENT_API_URL= for a local docs index (Phase 2). set -euo pipefail -cd "$(dirname "$0")/.." -ROOT="$PWD" +cd "$(dirname "$0")/../.." -[ -e evals/.env ] || { echo "evals/.env missing — run: mise run setup" >&2; exit 1; } -source scripts/load-keys.sh -cd evals && SUPABASE_MCP_SERVER_PATH="$ROOT/mcp/packages/mcp-server-supabase" pnpm eval "$@" +SUPABASE_MCP_SERVER_PATH="submodules/mcp/packages/mcp-server-supabase" exec workspace/scripts/eval.sh "$@" diff --git a/workspace/scripts/patches-lib.sh b/workspace/scripts/patches-lib.sh index a0146b78..b2c5d700 100644 --- a/workspace/scripts/patches-lib.sh +++ b/workspace/scripts/patches-lib.sh @@ -7,8 +7,11 @@ _MANIFEST_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) _manifest() { node "$_MANIFEST_LIB_DIR/manifest.mjs" "$@"; } -# PATCH_REPOS = repos carrying enabler patches (manifest order) -# PUBLISH_REPOS = every publishable editing surface (patch repos + submodules) +# PATCH_REPOS = repos carrying enabler patches (manifest order): mcp, supabase +# PUBLISH_REPOS = same set here — skills has no patches and is neither a patch +# nor a publish repo in this manifest (both derive from the +# same "has patches" predicate; kept as separate names because +# downstream scripts read them for different purposes) # # Fail-loud init: `for x in $(failing-cmd)` does NOT trip `set -e` in the # sourcing script — the substitution failure is swallowed and the lists come @@ -20,13 +23,12 @@ if [ -z "$_MANIFEST_REPOS" ]; then exit 1 fi PATCH_REPOS="" -PUBLISH_REPOS="" for _r in $_MANIFEST_REPOS; do - PUBLISH_REPOS="${PUBLISH_REPOS:+$PUBLISH_REPOS }$_r" if [ -n "$(_manifest get "$_r" patches)" ]; then PATCH_REPOS="${PATCH_REPOS:+$PATCH_REPOS }$_r" fi done +PUBLISH_REPOS="$PATCH_REPOS" unset _r _MANIFEST_REPOS repo_dir() { local d; d=$(_manifest get "$1" dir); echo "${d:-$1}"; } @@ -35,7 +37,7 @@ repo_remote() { _manifest get "$1" remote; } patches_for() { local out="" n for n in $(_manifest get "$1" patches); do - out="${out:+$out }patches/$n.patch" + out="${out:+$out }workspace/patches/$n.patch" done echo "$out" } diff --git a/workspace/scripts/provenance.mjs b/workspace/scripts/provenance.mjs index 2e3d2960..292f3c4f 100755 --- a/workspace/scripts/provenance.mjs +++ b/workspace/scripts/provenance.mjs @@ -23,7 +23,11 @@ import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { loadManifest } from "./manifest.mjs"; -const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +// This is the evals repo root (workspace/scripts -> ../..): supabase/, +// submodules/, and .docs-index-stamp.json all live here. The tracked-patches +// directory is a level lower, under workspace/ (see workspaceRoot below). +const root = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const workspaceRoot = join(root, "workspace"); // The one manifest loader (read + schema validation) — imported, not spawned, // so this file and the manifest.mjs CLI structurally cannot diverge on how a // bad manifest fails. @@ -101,22 +105,50 @@ const repoState = (dir) => { }; }; +// The two pinned submodules, keyed the way ab.sh/status.sh --json expect +// (basename of the submodules/ path). `git submodule status` prefixes an +// uninitialized entry with "-" — its SHA there is only the pinned index +// entry, not a real checkout, so that reads as null, not a fake SHA. +const submoduleShas = () => { + const shas = { "agent-skills": null, mcp: null }; + for (const line of (git(".", "submodule", "status") ?? "").split("\n")) { + if (!line) continue; + const initialized = line[0] !== "-"; + const [sha, subPath] = line.slice(1).trim().split(/\s+/); + const name = subPath?.split("/").pop(); + if (name && name in shas) shas[name] = initialized ? sha : null; + } + return shas; +}; + // Receipt assembly is LAZY: only the paths that emit a receipt (print/--embed) // pay for it. --stamp-docs-index must not — repoState alone walks and hashes // every clone's untracked files. const buildProvenance = () => { + const hostState = repoState("."); + const host = { + checkout_sha: hostState.sha, + checkout_dirty: hostState.dirty, + dirty_diff_sha256: hostState.dirty_diff_sha256, + untracked: hostState.untracked, + }; + const submodules = submoduleShas(); + + // Only non-submodule entries (today: supabase) get a full clone-shaped + // `repos.` record — the two submodules are already covered above by + // exact pinned SHA, which is what a submodule actually promises. const repos = {}; const patches = {}; for (const [name, spec] of Object.entries(manifest.repos)) { - if (!existsSync(join(root, spec.dir, ".git"))) { - repos[name] = { dir: spec.dir, cloned: false }; - continue; + const initialized = existsSync(join(root, spec.dir, ".git")); + if (spec.kind !== "submodule") { + repos[name] = initialized ? { dir: spec.dir, cloned: true, ...repoState(spec.dir) } : { dir: spec.dir, cloned: false }; } - repos[name] = { dir: spec.dir, cloned: true, ...repoState(spec.dir) }; + if (!spec.patches?.length || !initialized) continue; const localSubjects = (git(spec.dir, "log", "--format=%s", "HEAD", "--not", "--remotes") ?? "").split("\n"); - for (const p of spec.patches ?? []) { - const file = join(root, "patches", `${p}.patch`); + for (const p of spec.patches) { + const file = join(workspaceRoot, "patches", `${p}.patch`); const kind = (spec.localPatches ?? []).includes(p) ? "local" : "upstream"; patches[p] = { repo: name, @@ -128,10 +160,10 @@ const buildProvenance = () => { } // SUPABASE_MCP_SERVER_PATH swaps the mcp server for an arbitrary local - // build, so a receipt that only records the mcp CLONE's SHA would mislabel - // the arm. Record the override target verbatim plus, when it lives inside a - // git checkout (the workspace clone, the evals submodule, anywhere), that - // checkout's exact HEAD and dirty state. + // build, so a receipt that only records the mcp submodule's pinned SHA + // would mislabel the arm. Record the override target verbatim plus, when + // it lives inside a git checkout (the mcp submodule, an out-of-tree build, + // anywhere), that checkout's exact HEAD and dirty state. let mcp_override = null; const overridePath = process.env.SUPABASE_MCP_SERVER_PATH; if (overridePath) { @@ -159,7 +191,7 @@ const buildProvenance = () => { } } - return { generated_at: new Date().toISOString(), repos, patches, mcp_override, docs_index }; + return { generated_at: new Date().toISOString(), host, submodules, repos, patches, mcp_override, docs_index }; }; // Combined fingerprint of the supabase enabler-patch set: part of the docs @@ -168,7 +200,7 @@ const supabasePatchSetSha256 = () => { const names = manifest.repos.supabase?.patches ?? []; const h = createHash("sha256"); for (const p of names) { - const file = join(root, "patches", `${p}.patch`); + const file = join(workspaceRoot, "patches", `${p}.patch`); if (!existsSync(file)) { throw new Error(`manifest lists supabase patch "${p}" but ${file} is missing — manifest/patches drift`); } diff --git a/workspace/scripts/publish.sh b/workspace/scripts/publish.sh index e6dbcd06..3b67a5c5 100755 --- a/workspace/scripts/publish.sh +++ b/workspace/scripts/publish.sh @@ -8,17 +8,17 @@ # blocks any marker commit from being pushed by accident. # # Usage: -# scripts/publish.sh --list what's publishable -# scripts/publish.sh [--with ]... -# : evals | mcp | supabase | skills (the agent-skills submodule) +# workspace/scripts/publish.sh --list what's publishable +# workspace/scripts/publish.sh [--with ]... +# : mcp (submodule) | supabase (clone) — the only publishable repos set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." ROOT="$PWD" -source scripts/patches-lib.sh +source workspace/scripts/patches-lib.sh repo="${1:-}"; shift || true case " $PUBLISH_REPOS " in *" $repo "*) ;; *) - echo "usage: mise run publish [--with ]... | --list" >&2; exit 2 ;; + echo "usage: mise run publish [--with ]... | --list" >&2; exit 2 ;; esac dir=$(repo_dir "$repo") [ -e "$dir/.git" ] || { echo "$dir not cloned" >&2; exit 1; } diff --git a/workspace/scripts/setup.sh b/workspace/scripts/setup.sh index 100a21ba..d5e5a987 100755 --- a/workspace/scripts/setup.sh +++ b/workspace/scripts/setup.sh @@ -1,12 +1,9 @@ #!/usr/bin/env bash -# Clone + wire the repos needed for the skills loop, install deps, print status. -# Idempotent: existing clones are left alone (pull them yourself). +# Install deps, init submodules (agent-skills + mcp), apply patches, wire +# .env, print status. Idempotent: safe to re-run any time. set -euo pipefail -cd "$(dirname "$0")/.." -source scripts/patches-lib.sh - -EVALS_REMOTE="${EVALS_REMOTE:-$(repo_remote evals)}" +cd "$(dirname "$0")/../.." log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } warn() { printf '\033[1;33mwarn:\033[0m %s\n' "$*" >&2; } @@ -21,6 +18,22 @@ else warn "docker not found — sandbox-runtime evals need it (tools-mode evals don't)." fi +# --- deps --- +if [ -d node_modules ]; then + log "node_modules present — skipping install." +else + log "Installing deps (corepack + pnpm install)…" + corepack enable >/dev/null 2>&1 || true + pnpm install +fi + +# --- submodules: agent-skills + mcp (supabase clone stays opt-in) --- +log "Syncing submodules (agent-skills, mcp)…" +git submodule update --init submodules/agent-skills submodules/mcp + +# --- enabler plumbing: marker commits + pre-push guards (idempotent) --- +workspace/scripts/apply-patches.sh + # --- .env --- if [ -f .env ]; then log ".env already exists — leaving it." @@ -28,31 +41,10 @@ else cp .env.example .env log "Created .env from .env.example — fill in your API keys." fi - -# --- clone evals (with agent-skills submodule) --- -if [ -e evals/.git ]; then - log "evals/ already cloned — skipping." -else - log "Cloning evals (with agent-skills submodule)…" - git clone --recurse-submodules "$EVALS_REMOTE" evals +if [ "$(uname -s)" = Darwin ]; then + log "Tip: store keys in the keychain instead of .env — mise run store-key " fi -# Self-heal: ensure the agent-skills submodule is present even if evals was -# cloned earlier without --recurse-submodules (or a clone was interrupted). -log "Syncing submodules…" -git -C evals submodule update --init --recursive - -# --- share one .env with evals --- -ln -sfn ../.env evals/.env -log "Symlinked .env → evals/.env" - -# --- install deps --- -log "Installing evals deps (pnpm install)…" -( cd evals && pnpm install ) - -# --- enabler plumbing: marker commits + pre-push guards (idempotent) --- -scripts/apply-patches.sh - -log "Setup done." +log "Setup done. (Docs loop is optional and heavy — see: mise run clone-docs)" echo -scripts/status.sh +workspace/scripts/status.sh diff --git a/workspace/scripts/status.sh b/workspace/scripts/status.sh index 3ea35ed5..247f9ee4 100755 --- a/workspace/scripts/status.sh +++ b/workspace/scripts/status.sh @@ -1,18 +1,19 @@ #!/usr/bin/env bash -# Workspace status: per-repo branch/SHA/dirty, env keys present, tooling. +# Workspace status: host repo + submodule + clone state, env keys present, tooling. set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." -# --json: machine-readable provenance receipt (repos, patches, docs stamp) -if [ "${1:-}" = "--json" ]; then exec node scripts/provenance.mjs; fi -source scripts/patches-lib.sh +# --json: machine-readable provenance receipt (host, submodules, repos, patches, docs stamp) +if [ "${1:-}" = "--json" ]; then exec node workspace/scripts/provenance.mjs; fi +source workspace/scripts/patches-lib.sh -# label, path. `.git` is a dir for clones and a file for submodules, so -e covers both. +# label, dir, [missing-message]. `.git` is a dir for clones and a file for +# submodules, so -e covers both. repo_status() { - local label="$1" dir="$2" branch sha dirty + local label="$1" dir="$2" missing="${3:-not cloned}" branch sha dirty if [ ! -e "$dir/.git" ]; then - printf ' %-22s not cloned\n' "$label" + printf ' %-22s %s\n' "$label" "$missing" return fi branch=$(git -C "$dir" rev-parse --abbrev-ref HEAD 2>/dev/null || echo '?') @@ -23,33 +24,38 @@ repo_status() { NEXT="" echo "Repos:" -# Rows come from the manifest (PUBLISH_REPOS order); agent-skills is the one -# hand-nested row, shown under its host clone evals. +# The host repo is this checkout itself (evals); agent-skills and mcp are its +# two pinned submodules (hand-nested rows); supabase is the only opt-in clone. +repo_status "evals (host)" "." +repo_status " - agent-skills" "$(repo_dir skills)" "not initialized" +repo_status " - mcp" "$(repo_dir mcp)" "not initialized" for _repo in $PUBLISH_REPOS; do - [ "$_repo" = skills ] && continue + case "$_repo" in skills|mcp) continue ;; esac repo_status "$_repo" "$(repo_dir "$_repo")" - if [ "$_repo" = evals ]; then - repo_status " - agent-skills" "$(repo_dir skills)" - fi done unset _repo -# setup.sh also repairs a missing agent-skills submodule, root .env, and the -# evals/.env symlink — suggest it if ANY of those is missing (not just the clone). -if [ ! -e evals/.git ] || [ ! -e evals/submodules/agent-skills/.git ] || [ ! -f .env ] || [ ! -e evals/.env ]; then - NEXT="${NEXT} mise run setup # clone/repair evals (+ agent-skills), install, wire .env\n" + +# Bootstrap order (mirrors mise run setup): deps, then submodules, then +# enabler plumbing, then keys. The supabase clone is opt-in for the docs loop +# only — its absence is informational (shown above) and never gates Ready. +[ -d node_modules ] || NEXT="${NEXT} pnpm install # install workspace deps\n" +if [ ! -e "$(repo_dir skills)/.git" ] || [ ! -e "$(repo_dir mcp)/.git" ]; then + NEXT="${NEXT} git submodule update --init submodules/agent-skills submodules/mcp # init submodules\n" fi -# enabler plumbing present? (marker commits; see apply-patches.sh) +# enabler plumbing present? (marker commits; see apply-patches.sh) — only +# checked for repos that are actually present (supabase clone, mcp submodule). for _r in $PATCH_REPOS; do - [ -e "$_r/.git" ] || continue - _subjects=$(git -C "$_r" log --format=%s HEAD --not --remotes 2>/dev/null || true) + _d=$(repo_dir "$_r") + [ -e "$_d/.git" ] || continue + _subjects=$(git -C "$_d" log --format=%s HEAD --not --remotes 2>/dev/null || true) for _p in $(patches_for "$_r"); do if ! printf '%s\n' "$_subjects" | grep -qxF "$(patch_subject "$_p")"; then - NEXT="${NEXT} mise run apply-patches # enabler plumbing commit(s) missing in $_r\n" + NEXT="${NEXT} mise run apply-patches # enabler plumbing commit(s) missing in $_d\n" break fi done done -# pre-push guards active? (apply-patches installs them) +# pre-push guards active? (apply-patches installs them, incl. in agent-skills) for _n in $PUBLISH_REPOS; do _d=$(repo_dir "$_n") [ -e "$_d/.git" ] || continue @@ -58,12 +64,10 @@ for _n in $PUBLISH_REPOS; do NEXT="${NEXT} mise run apply-patches # pre-push guard missing in $_d\n" fi done +unset _r _n _d _subjects _p _hooks echo echo "Credentials (source per key; ANTHROPIC + OPENAI required, GEMINI optional):" -if [ ! -e evals/.env ]; then - echo " ! evals/.env missing — node --env-file will fail; run: mise run setup" -fi IS_DARWIN=""; [ "$(uname -s)" = Darwin ] && IS_DARWIN=1 for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do src="" @@ -96,5 +100,5 @@ else echo "Ready. Try:" echo " mise run ab-test # zero-cost self-check of the A/B runner" echo " mise run ab # A/B readiness probe (head-to-head evals)" - echo " mise run eval -- --eval --experiment # run an eval (see evals/evals/)" + echo " mise run eval -- --eval --experiment # run an eval (see evals/)" fi diff --git a/workspace/scripts/status.test.sh b/workspace/scripts/status.test.sh index d809357f..93e5c616 100755 --- a/workspace/scripts/status.test.sh +++ b/workspace/scripts/status.test.sh @@ -3,7 +3,7 @@ # against synthetic workspace states in a temp dir (never touches this one). # USER=nobody forces keychain misses so results don't depend on real keys. set -euo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." SRC="$PWD" pass=0; fail=0 @@ -11,42 +11,54 @@ ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "F run_case() { # $1 = label, $2 = state-builder function; output -> /tmp/status_case.out local tmp; tmp="$(mktemp -d /tmp/eval-ws-status-XXXXXX)" - mkdir -p "$tmp/scripts" "$tmp/bin" - cp "$SRC/scripts/status.sh" "$SRC/scripts/patches-lib.sh" "$SRC/scripts/manifest.mjs" "$tmp/scripts/" - cp "$SRC/manifest.json" "$SRC/.env.example" "$tmp/" + mkdir -p "$tmp/workspace/scripts" "$tmp/bin" + cp "$SRC/workspace/scripts/status.sh" "$SRC/workspace/scripts/patches-lib.sh" "$SRC/workspace/scripts/manifest.mjs" "$tmp/workspace/scripts/" + cp "$SRC/workspace/manifest.json" "$tmp/workspace/" + cp "$SRC/.env.example" "$tmp/" + # This checkout IS the host repo in real usage — fake the marker so the + # host row never misreports "not cloned" while a case exercises other gaps. + mkdir -p "$tmp/.git" # OS-deterministic: default to Darwin regardless of host; a case may overwrite bin/uname. printf '#!/bin/sh\necho Darwin\n' > "$tmp/bin/uname"; chmod +x "$tmp/bin/uname" ( cd "$tmp" && "$2" ) - ( cd "$tmp" && USER=nobody PATH="$PWD/bin:$PATH" bash scripts/status.sh 2>/dev/null ) > /tmp/status_case.out + ( cd "$tmp" && USER=nobody PATH="$PWD/bin:$PATH" bash workspace/scripts/status.sh 2>/dev/null ) > /tmp/status_case.out rm -rf "$tmp" } has() { grep -qF -- "$1" /tmp/status_case.out && echo y || echo n; } -# --- fresh clone: nothing set up --- +# --- fresh: nothing set up --- state_fresh() { :; } run_case fresh state_fresh -ck "fresh: suggests setup" "$(has 'mise run setup')" y -ck "fresh: suggests required keys" "$(has 'mise run store-key OPENAI_API_KEY')" y -ck "fresh: not Ready" "$(has 'Ready.')" n - -# --- partial: evals cloned, submodule + evals/.env missing (interrupted setup) --- -state_partial_submodule() { mkdir -p evals/.git; cp .env.example .env; } -run_case partial-submodule state_partial_submodule -ck "partial submodule: suggests setup" "$(has 'mise run setup')" y -ck "partial submodule: not Ready" "$(has 'Ready.')" n - -# --- partial: clone + submodule fine, evals/.env symlink missing --- -state_partial_env() { mkdir -p evals/.git evals/submodules/agent-skills/.git; cp .env.example .env; } -run_case partial-env state_partial_env -ck "partial env: suggests setup" "$(has 'mise run setup')" y -ck "partial env: not Ready" "$(has 'Ready.')" n - -# --- complete workspace, only keys missing: setup NOT suggested, keys are --- -state_keys_only() { mkdir -p evals/.git evals/submodules/agent-skills/.git; cp .env.example .env; ln -s ../.env evals/.env; } +ck "fresh: suggests pnpm install" "$(has 'pnpm install')" y +ck "fresh: suggests submodule init" "$(has 'submodule update --init')" y +ck "fresh: suggests required keys" "$(has 'mise run store-key OPENAI_API_KEY')" y +ck "fresh: not Ready" "$(has 'Ready.')" n + +# --- deps installed, submodules not yet initialized --- +state_deps_only() { mkdir -p node_modules; cp .env.example .env; } +run_case deps-only state_deps_only +ck "deps-only: no pnpm-install suggestion" "$(has 'pnpm install')" n +ck "deps-only: suggests submodule init" "$(has 'submodule update --init')" y +ck "deps-only: not Ready" "$(has 'Ready.')" n + +# --- submodules initialized (markers/hooks still missing), env missing --- +state_submodules_init() { mkdir -p node_modules submodules/agent-skills/.git submodules/mcp/.git; } +run_case submodules-init state_submodules_init +ck "submodules-init: no pnpm-install suggestion" "$(has 'pnpm install')" n +ck "submodules-init: no submodule-init suggestion" "$(has 'submodule update --init')" n +ck "submodules-init: suggests required keys" "$(has 'mise run store-key ANTHROPIC_API_KEY')" y +ck "submodules-init: not Ready" "$(has 'Ready.')" n +# supabase was never cloned in this state — its absence must stay informational, +# never surfacing an action item (there is no clone-docs step in the bootstrap order). +ck "submodules-init: supabase absence isn't an action item" "$(has 'clone-docs')" n + +# --- complete workspace state (deps + submodules + env), only plumbing/keys stay open --- +state_keys_only() { mkdir -p node_modules submodules/agent-skills/.git submodules/mcp/.git; cp .env.example .env; } run_case keys-only state_keys_only -ck "keys-only: no setup suggestion" "$(has 'mise run setup')" n -ck "keys-only: suggests keys" "$(has 'mise run store-key ANTHROPIC_API_KEY')" y -ck "keys-only: not Ready" "$(has 'Ready.')" n +ck "keys-only: no pnpm-install suggestion" "$(has 'pnpm install')" n +ck "keys-only: no submodule-init suggestion" "$(has 'submodule update --init')" n +ck "keys-only: suggests keys" "$(has 'mise run store-key ANTHROPIC_API_KEY')" y +ck "keys-only: not Ready" "$(has 'Ready.')" n # --- non-Darwin: same complete-but-keyless state; keys must route to .env, never store-key --- state_linux_keys() { state_keys_only; printf '#!/bin/sh\necho Linux\n' > bin/uname; } diff --git a/workspace/scripts/update.sh b/workspace/scripts/update.sh index d2748100..c4514e57 100755 --- a/workspace/scripts/update.sh +++ b/workspace/scripts/update.sh @@ -1,16 +1,21 @@ #!/usr/bin/env bash -# Update the clones: fetch upstream and REBASE local work — the [eval-workspace-*] -# plumbing commits plus any commits of yours — onto the new tip. Uncommitted -# edits survive via --autostash. A rebase conflict means upstream drifted under -# a plumbing commit (regenerate that patch; see patches/README.md) or under -# your own commits; the rebase is aborted so the workspace is left as found. +# Update the supabase clone: fetch upstream and REBASE local work — the +# [eval-workspace-*] plumbing commits plus any commits of yours — onto the +# new tip. Uncommitted edits survive via --autostash. A rebase conflict means +# upstream drifted under a plumbing commit (regenerate that patch; see +# workspace/patches/README.md) or under your own commits; the rebase is +# aborted so the workspace is left as found. # -# Usage: scripts/update.sh [--check] [repo...] +# mcp and skills are pin-driven submodules (no remote in manifest.json) — +# not managed here. Bump the pin with `git submodule update --remote ` +# and record it via the M3 pin flow, then re-run `mise run apply-patches`. +# +# Usage: workspace/scripts/update.sh [--check] [repo...] # --check fetch + report how far behind each clone is; changes nothing # repos default: every patch-carrying repo from manifest.json set -euo pipefail -cd "$(dirname "$0")/.." -source scripts/patches-lib.sh +cd "$(dirname "$0")/../.." +source workspace/scripts/patches-lib.sh CHECK=0 [ "${1:-}" = "--check" ] && { CHECK=1; shift; } @@ -19,71 +24,52 @@ REPOS=("$@") FAILED=0 for repo in ${REPOS[@]+"${REPOS[@]}"}; do - if [ ! -e "$repo/.git" ]; then echo "== $repo: not cloned — skipping"; continue; fi - branch=$(git -C "$repo" rev-parse --abbrev-ref HEAD) - old_upstream=$(git -C "$repo" rev-parse "origin/$branch" 2>/dev/null || echo "") - git -C "$repo" fetch -q - behind=$(git -C "$repo" rev-list --count "HEAD..origin/$branch" 2>/dev/null || echo '?') + dir=$(repo_dir "$repo") + remote=$(repo_remote "$repo") + if [ -z "$remote" ]; then + echo "== $repo: pin-driven ($dir) — not fetched/rebased here." + echo " bump: git submodule update --remote $dir (then record the pin — see M3 pin flow)" + continue + fi + if [ ! -e "$dir/.git" ]; then echo "== $repo: not cloned — skipping"; continue; fi + branch=$(git -C "$dir" rev-parse --abbrev-ref HEAD) + old_upstream=$(git -C "$dir" rev-parse "origin/$branch" 2>/dev/null || echo "") + git -C "$dir" fetch -q + behind=$(git -C "$dir" rev-list --count "HEAD..origin/$branch" 2>/dev/null || echo '?') if [ "$CHECK" = 1 ]; then - printf '== %-9s %s@%s is %s commit(s) behind origin/%s\n' "$repo:" "$branch" "$(git -C "$repo" rev-parse --short HEAD)" "$behind" "$branch" + printf '== %-9s %s@%s is %s commit(s) behind origin/%s\n' "$repo:" "$branch" "$(git -C "$dir" rev-parse --short HEAD)" "$behind" "$branch" continue fi echo "== $repo: $branch, $behind commit(s) behind — updating" - git -C "$repo" diff --cached --quiet --ita-visible-in-index \ - || { echo " $repo index has staged changes — unstage or commit first" >&2; FAILED=1; continue; } + git -C "$dir" diff --cached --quiet --ita-visible-in-index \ + || { echo " $dir index has staged changes — unstage or commit first" >&2; FAILED=1; continue; } if [ "$behind" = 0 ]; then echo " already up to date" continue fi - # capture agent-skills state BEFORE the rebase moves the recorded pin - if [ "$repo" = evals ]; then - sk=evals/submodules/agent-skills - old_pin=$(git -C evals ls-tree HEAD submodules/agent-skills | awk '{print $3}') - sk_head=$(git -C "$sk" rev-parse HEAD 2>/dev/null || echo "") - sk_clean=$([ -e "$sk/.git" ] && [ -z "$(git -C "$sk" status --porcelain 2>/dev/null)" ] && echo yes || echo no) - fi - - if ! git -C "$repo" rebase --autostash -q "origin/$branch"; then - git -C "$repo" rebase --abort 2>/dev/null || true + if ! git -C "$dir" rebase --autostash -q "origin/$branch"; then + git -C "$dir" rebase --abort 2>/dev/null || true echo " REBASE CONFLICT: upstream drifted under a plumbing commit or your work — nothing changed." >&2 - echo " Fix: regenerate the conflicting patch (patches/README.md) or rebase manually in $repo/." >&2 + echo " Fix: regenerate the conflicting patch (workspace/patches/README.md) or rebase manually in $dir/." >&2 FAILED=1 continue fi - echo " now at $(git -C "$repo" rev-parse --short HEAD) (upstream $(git -C "$repo" rev-parse --short "origin/$branch"))" + echo " now at $(git -C "$dir" rev-parse --short HEAD) (upstream $(git -C "$dir" rev-parse --short "origin/$branch"))" - if [ "$repo" = evals ] && [ -e "$sk/.git" ]; then - # sync the pin only when skills had NO local work before the update: - # clean tree AND sitting exactly on the previously recorded gitlink - if [ "$sk_clean" = yes ] && [ "$sk_head" = "$old_pin" ]; then - git -C evals submodule update --init --recursive -q - else - echo " note: agent-skills submodule has local work (edits or commits) — skipping submodule sync" - fi - fi - - new_upstream=$(git -C "$repo" rev-parse "origin/$branch") + new_upstream=$(git -C "$dir" rev-parse "origin/$branch") if [ "$old_upstream" != "$new_upstream" ]; then case "$repo" in - evals) - if ! git -C evals diff --quiet "$old_upstream" "$new_upstream" -- pnpm-lock.yaml; then - echo " lockfile changed — pnpm install"; ( cd evals && pnpm install --silent ) - fi ;; supabase) - if ! git -C supabase diff --quiet "$old_upstream" "$new_upstream" -- pnpm-lock.yaml; then - echo " lockfile changed — pnpm install (docs filter)"; pnpm --dir supabase install --filter docs... --silent + if ! git -C "$dir" diff --quiet "$old_upstream" "$new_upstream" -- pnpm-lock.yaml; then + echo " lockfile changed — pnpm install (docs filter)"; pnpm --dir "$dir" install --filter docs... --silent fi - if [ -n "$(git -C supabase diff --name-only "$old_upstream" "$new_upstream" -- apps/docs/content | head -1)" ]; then + if [ -n "$(git -C "$dir" diff --name-only "$old_upstream" "$new_upstream" -- apps/docs/content | head -1)" ]; then echo " note: docs content changed upstream — the next docs-index re-embeds changed pages (costs cents)" fi ;; - mcp) - if [ -d mcp/packages/mcp-server-supabase/dist ]; then - echo " rebuilding local mcp"; ( cd mcp && pnpm install --silent && pnpm build ) - fi ;; esac fi done From b36ab40bbb6fcf8b9a0374ee6ef10d048d25b301 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 09:49:31 +0200 Subject: [PATCH 4/9] docs+ci: workspace README, patches manifest for the folded layout, free self-test workflow workspace/README.md is the ops doc (bootstrap, the three loops, A/B, provenance, self-tests); the root README points at it. The self-test workflow runs the three shell-only suites on workspace/** PRs: no keys, no model spend, no docker. --- .github/workflows/workspace-selftest.yml | 31 +++++++++ README.md | 10 +++ workspace/README.md | 85 ++++++++++++++++++++++++ workspace/patches/README.md | 38 ++++++----- 4 files changed, 146 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/workspace-selftest.yml create mode 100644 workspace/README.md diff --git a/.github/workflows/workspace-selftest.yml b/.github/workflows/workspace-selftest.yml new file mode 100644 index 00000000..cc1bc847 --- /dev/null +++ b/.github/workflows/workspace-selftest.yml @@ -0,0 +1,31 @@ +name: workspace-selftest + +# Free self-tests of the eval-source workspace glue (workspace/README.md): +# shell + node stdlib only — no API keys, no model spend, no docker, no +# pnpm install. Submodules are fetched so the hooks/ab tests exercise the +# real repos instead of skipping. +on: + pull_request: + paths: + - "workspace/**" + - "mise.toml" + - ".github/workflows/workspace-selftest.yml" + +jobs: + selftest: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + submodules: recursive + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: status self-test + run: bash workspace/scripts/status.test.sh + - name: hooks self-test + run: bash workspace/scripts/hooks.test.sh + - name: ab self-test + run: bash workspace/scripts/ab.test.sh diff --git a/README.md b/README.md index fb1e021c..d2d13676 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,16 @@ Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/co - **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network) under `.claude/skills/`. When a task matches, the agent reads `.claude/skills//SKILL.md` (and any files it references) with its file tools. - **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name. +## Eval-source workspace (docs / skills / MCP loops) + +The glue for testing changes to the agent's *inputs* — docs pages, skills, +and MCP server source — lives in [`workspace/`](workspace/README.md): edit a +source, run the affected evals against the local change, or run a head-to-head +A/B (`mise run ab`) with per-arm provenance receipts. Tasks are driven by +`mise` from the repo root (`mise run status` is the bootstrap probe). The +docs monorepo stays an opt-in sparse clone; the MCP server is the pinned +`submodules/mcp` (see "Running against an exact MCP server revision" above). + ## Framework Checks ```bash diff --git a/workspace/README.md b/workspace/README.md new file mode 100644 index 00000000..cb69e169 --- /dev/null +++ b/workspace/README.md @@ -0,0 +1,85 @@ +# Eval-source workspace + +Glue that wires the agent's **inputs** — docs, skills, and the MCP server — +into this harness, so you can change an input and measure the effect on evals: +edit a skill, a docs page, or MCP server source, then run the affected evals +against the local change. Formerly the standalone +[eval-workspace](https://github.com/supabase/eval-workspace) repo; folded in +here so the sources under test live beside the harness that tests them +(direction agreed in the workspace-layout Slack thread, 2026-07-23). + +All tasks run via `mise` from the repo root (Supabase convention; `mise.toml` +lives there). Flag-style args need `--`: `mise run eval -- --eval `. + +## Bootstrap + +```bash +mise run status # the state probe: prints the exact next command for anything missing +mise run setup # idempotent: install, init submodules (agent-skills + mcp), patches, .env +``` + +Keys live in the macOS keychain as `eval-workspace:` — add them with +`mise run store-key ` (hidden +prompt, immune to the 128-char truncation of raw `security -w`). Non-macOS: +put keys in the repo-root `.env` (the fallback `status` will route you to). + +`mise run ab-test` is the zero-cost self-check that the glue works. + +## The three loops + +| Loop | Source | Sync after an edit | +|---|---|---| +| Skills | `submodules/agent-skills` (a working tree: edit in place) | none | +| MCP server | `submodules/mcp` | `mise run mcp-build` | +| Docs | `supabase/` (opt-in sparse clone: `mise run clone-docs`) | `mise run docs-index` (cents) | + +- **MCP loop**: `mise run mcp-eval -- ` builds the submodule (with the + enabler patches) and runs evals against it via `SUPABASE_MCP_SERVER_PATH`. +- **Docs loop** (heavy: Docker + supabase CLI): `docs-up` → `docs-seed` (spends + OpenAI money once, ~$0.12; always confirm with the user first) → `docs-api` + serves the content GraphQL API for `search_docs`; `docs-index` re-embeds + changed pages incrementally. Content DB ports are 55321+ to avoid the eval + local-stack range (54321-9). Measuring docs impact needs a tools-mode + (`interface: mcp`) eval whose answer lives only in the docs. +- `mise run affected -- ` maps changed source paths to a ready + `mise run eval …` line. + +## Head-to-head A/B + +Measure whether ONE edit moves an eval: make a tracked, unstaged edit in a +loop's scope, then + +```bash +mise run ab # treatment (edit applied) vs baseline (edit reverted) +``` + +The edit is always restored (a failed restore fails the run loudly); per-arm +provenance receipts land in `results-ab/*.json`. Cost: two model runs. No +args = readiness probe. First time? `mise run ab-demo` is a guided, +self-cleaning live proof on the docs loop (spend-gated, asks first). + +## Patches & publishing + +Local changes to the patched repos (the `supabase/` clone and the +`submodules/mcp` working tree) are tracked as `.patch` files in +`workspace/patches/` and applied as marker commits — see +[patches/README.md](./patches/README.md) for the manifest and the +publish flow (`mise run publish `). A pre-push guard in each +patched repo blocks marker commits from leaving the machine; the host repo +never gets hooks or marker commits. + +## Provenance + +`mise run status -- --json` prints a receipt: host repo SHA + dirty state, +submodule pins, supabase clone state, patch fingerprints, and the docs-index +stamp (`.docs-index-stamp.json`, scoped to repo docs content only). `ab.sh` +embeds a per-arm copy into every A/B result, so a wrong-baseline run is +immediately obvious. + +## Self-tests (all free: no keys, no model spend) + +```bash +mise run status-test # status.sh Next/Ready diagnosis against synthetic states +mise run hooks-test # pre-push guard lifecycle +mise run ab-test # A/B runner with a faked eval +``` diff --git a/workspace/patches/README.md b/workspace/patches/README.md index 88604275..420778fe 100644 --- a/workspace/patches/README.md +++ b/workspace/patches/README.md @@ -1,7 +1,9 @@ # Enabler patches -Local changes to the cloned repos, materialized by `scripts/apply-patches.sh` -as identifiable **local commits** at the bottom of each clone's branch: +Local changes to the patched repos (the `supabase/` clone and the +`submodules/mcp` working tree), materialized by +`workspace/scripts/apply-patches.sh` as identifiable **local commits** at the +bottom of each repo's branch: [eval-workspace-local] dev shim — must never leave this machine [eval-workspace-upstream] upstream candidate — leaves ONLY via @@ -9,23 +11,22 @@ as identifiable **local commits** at the bottom of each clone's branch: Your own work sits **above** these as normal commits, so `git commit -am` can never sweep plumbing into it, and `mise run ab` can A/B any file (plumbing is -not in the working diff). A pre-push guard in every clone (and the agent-skills -submodule) blocks marker commits from being pushed; a pre-existing pre-push -hook is chained after the guard. Deliberate override (skips only the marker -checks): `EVAL_WORKSPACE_ALLOW_PUSH=1 git push …`. +not in the working diff). A pre-push guard in each patched repo blocks marker +commits from being pushed; a pre-existing pre-push hook is chained after the +guard. The host repo (this one) never gets hooks or marker commits. Deliberate +override (skips only the marker checks): `EVAL_WORKSPACE_ALLOW_PUSH=1 git push …`. The `.patch` files here are **canonical**: `apply-patches` builds each commit from the patch via the index, verifies the staged diff equals the patch before committing (a user edit in a patch-owned file can never be absorbed), and on every later run verifies the existing marker commit still matches the file. -`manifest.json` is the single source of truth mapping patches → repos / kinds -(read via `scripts/manifest.mjs`; marker subjects derive from the kind). +`workspace/manifest.json` is the single source of truth mapping patches → +repos / kinds (read via `workspace/scripts/manifest.mjs`; marker subjects +derive from the kind). | Patch | Repo | Files | Kind | Tests | What | |---|---|---|---|---|---| -| `mcp-content-api-url` | supabase/mcp | `transports/stdio.ts` | upstream | 3 stdio integration tests (in the PR) | `--content-api-url` flag + `SUPABASE_CONTENT_API_URL` — **PR open: [mcp#343](https://github.com/supabase/mcp/pull/343)** | -| `evals-mcp-local-build-override` | supabase/evals | `core/index.ts`, `core/mcp-server.test.ts` | upstream | 4 `createConfig` tests | `SUPABASE_MCP_SERVER_PATH` local-build override (independent of the mcp flag; ships with the M1 submodule PR) | -| `evals-mcp-content-api-url` | supabase/evals | `core/index.ts`, `core/mcp-server.test.ts` | upstream | 2 `createConfig` tests | `contentApiUrl` option + `SUPABASE_CONTENT_API_URL` threading (applies on top of the override patch; upstream only after mcp's `--content-api-url` flag lands) | +| `mcp-content-api-url` | `submodules/mcp` | `transports/stdio.ts` | upstream | 3 stdio integration tests (in the PR) | `--content-api-url` flag + `SUPABASE_CONTENT_API_URL` — **PR open: [mcp#343](https://github.com/supabase/mcp/pull/343)**; the patch retires when it merges and the pin moves past it | | `supabase-content-local-ports` | supabase/supabase | `config.toml` | **LOCAL-ONLY** | n/a | dev ports 55321+ (avoid evals' 54321-9) | | `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report | | `supabase-docs-guide-checksum` | supabase/supabase | `guideModelLoader.ts` | upstream | none | guide checksum + `tryCatch` `onError` fix (docs-index edit detection) | @@ -39,18 +40,19 @@ verifies both directions and fails loudly on drift. - **You changed the plumbing in the clone** (amended/edited the marker commit): refresh the canonical file from the commit — - `git -C diff ^ > patches/.patch` + `git -C diff ^ > workspace/patches/.patch` (plain `git diff` cannot see committed changes). - **You edited the `.patch` file**: drop the old marker commit - (`git -C rebase --onto ^ `), then + (`git -C rebase --onto ^ `), then `mise run apply-patches` re-creates it from the file. - **A patch adds brand-new files**: when generating, `git add -N ` first, then `git reset -- ` after — a leftover intent-to-add entry breaks `git stash`. ## Upstreaming order -`evals-mcp-local-build-override` — independent of everything, shipped with the -evals M1 submodule PR (evals#109). Then `mcp-content-api-url` PR → -`evals-mcp-content-api-url` PR (depends on the flag) → the three upstream -supabase fixes (independent; `guide-checksum` and `index-fail-closed` still -need regression tests; `reference-dup-sources` is test-covered and PR-ready). +`mcp-content-api-url` — PR open ([mcp#343](https://github.com/supabase/mcp/pull/343)). +Then the three upstream supabase fixes (independent; `guide-checksum` and +`index-fail-closed` still need regression tests; `reference-dup-sources` is +test-covered and PR-ready). The two former evals patches are done: the +local-build override shipped with evals#109, and the contentApiUrl threading +landed as a normal commit on this branch. From 1010711f480166463ce36bc4c00cfb65e78ca4d0 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 10:15:11 +0200 Subject: [PATCH 5/9] fix: team-review round 1 (critical + important findings) - ab.test.sh cleanup could destroy a user's unstaged stdio.ts edit: the EXIT trap reverted the mcp fixture file unconditionally while the fixture block only runs on a CLEAN file - exactly the state where the revert nukes user work. Cleanup now only reverts what the test itself wrote (verified: planted user edit survives a skipped-fixture run). - fold demo/canary-eval (ab-demo died at cp: fixtures were never folded). - affected.ts emits source-selecting commands: mcp/server-wide paths get mise run mcp-eval; docs paths get SUPABASE_CONTENT_API_URL + mcp-eval (a bare 'mise run eval' measured production, not the local edit). - workspace/README.md documents the docs-loop measurement step. - workspace-selftest.yml: pin setup-node to the repo's SHA convention, node-version-file instead of an inline 22. - dedupe the paid docs-embed preamble (docs-embed-env.sh sourced by docs-index/docs-seed) and provenance's untracked-hashing (one hashUntracked helper for receipt + stamp). Self-tests: status 19/0, hooks 8/0, ab 22/0. --- .github/workflows/workspace-selftest.yml | 4 +-- demo/canary-eval/EVAL.ts | 19 ++++++++++++ demo/canary-eval/PROMPT.md | 13 +++++++++ workspace/README.md | 12 ++++---- workspace/scripts/ab.test.sh | 6 +++- workspace/scripts/affected.ts | 13 +++++++-- workspace/scripts/docs-embed-env.sh | 22 ++++++++++++++ workspace/scripts/docs-index.sh | 19 +----------- workspace/scripts/docs-seed.sh | 19 +----------- workspace/scripts/provenance.mjs | 37 ++++++++++-------------- 10 files changed, 96 insertions(+), 68 deletions(-) create mode 100644 demo/canary-eval/EVAL.ts create mode 100644 demo/canary-eval/PROMPT.md create mode 100644 workspace/scripts/docs-embed-env.sh diff --git a/.github/workflows/workspace-selftest.yml b/.github/workflows/workspace-selftest.yml index cc1bc847..8549a2b6 100644 --- a/.github/workflows/workspace-selftest.yml +++ b/.github/workflows/workspace-selftest.yml @@ -20,9 +20,9 @@ jobs: uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 with: submodules: recursive - - uses: actions/setup-node@v4 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 22 + node-version-file: .node-version - name: status self-test run: bash workspace/scripts/status.test.sh - name: hooks self-test diff --git a/demo/canary-eval/EVAL.ts b/demo/canary-eval/EVAL.ts new file mode 100644 index 00000000..a0ca1162 --- /dev/null +++ b/demo/canary-eval/EVAL.ts @@ -0,0 +1,19 @@ +import { type CheckResult, type ToolScorer } from '@supabase/evals/core' + +// eval-workspace docs-discriminator demo: '@supabase/pinniped' is a fictional package +// planted ONLY in the local docs by scripts/ab-demo.sh. It cannot come from model +// priors or CLI scaffolding, so naming it proves the agent retrieved the answer +// from the local docs index — and removing it (the A/B baseline) proves the doc +// was the cause. +const scorer: ToolScorer = async (ctx) => { + const report = ctx.agentReport ?? '' + const checks: CheckResult[] = [ + { + name: 'named the docs-only package @supabase/pinniped', + passed: /pinniped/i.test(report), + }, + ] + return { passed: checks[0].passed, checks } +} + +export default scorer diff --git a/demo/canary-eval/PROMPT.md b/demo/canary-eval/PROMPT.md new file mode 100644 index 00000000..b3b97423 --- /dev/null +++ b/demo/canary-eval/PROMPT.md @@ -0,0 +1,13 @@ +--- +stage: investigate +suite: other +interface: mcp +product: + - edge-functions +topic: + - sdk +--- + +I'm building a Supabase-managed cron/queue worker on the internal "Nimbus" +runtime. Search the Supabase documentation and tell me the exact JavaScript +helper package it recommends for this case. Name the package precisely. diff --git a/workspace/README.md b/workspace/README.md index cb69e169..bae2074f 100644 --- a/workspace/README.md +++ b/workspace/README.md @@ -38,11 +38,13 @@ put keys in the repo-root `.env` (the fallback `status` will route you to). - **Docs loop** (heavy: Docker + supabase CLI): `docs-up` → `docs-seed` (spends OpenAI money once, ~$0.12; always confirm with the user first) → `docs-api` serves the content GraphQL API for `search_docs`; `docs-index` re-embeds - changed pages incrementally. Content DB ports are 55321+ to avoid the eval - local-stack range (54321-9). Measuring docs impact needs a tools-mode - (`interface: mcp`) eval whose answer lives only in the docs. -- `mise run affected -- ` maps changed source paths to a ready - `mise run eval …` line. + changed pages incrementally. To measure a docs edit, point an eval run at + the local index through the local server build: + `SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql mise run mcp-eval -- --eval ` + (or just use `mise run ab`, which wires this automatically). Content DB + ports are 55321+ to avoid the eval local-stack range (54321-9). Measuring + docs impact needs a tools-mode (`interface: mcp`) eval whose answer lives + only in the docs. ## Head-to-head A/B diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh index ef22c2ec..5cd33567 100755 --- a/workspace/scripts/ab.test.sh +++ b/workspace/scripts/ab.test.sh @@ -28,9 +28,12 @@ RES="results/$EXP/$EVAL.json" # only now (after selecting a clean file) arm cleanup: revert our own edit, drop fixtures MCP=submodules/mcp MCPREL=packages/mcp-server-supabase/src/transports/stdio.ts +MCP_MUTATED=0 # cleanup may only revert $MCPREL if THIS test wrote to it — a + # user's own unstaged edit there skips the fixture block below, + # and must survive the run untouched. cleanup() { git -C "$SK" checkout -q -- "$REL" 2>/dev/null || true - [ -e "$MCP/.git" ] && git -C "$MCP" checkout -q -- "$MCPREL" 2>/dev/null || true + [ "$MCP_MUTATED" = 1 ] && git -C "$MCP" checkout -q -- "$MCPREL" 2>/dev/null || true rm -f "$RES" "results-ab/$EVAL".*.json } trap cleanup EXIT @@ -96,6 +99,7 @@ ck "explains staged-index refusal" "$(grep -c 'staged changes' /tmp/ab_selftes # commit must be untouched on both success and mid-run failure --- if [ -e "$MCP/.git" ] && git -C "$MCP" diff --quiet -- "$MCPREL" && git -C "$MCP" diff --cached --quiet --ita-visible-in-index; then marker0=$(git -C "$MCP" rev-parse HEAD) + MCP_MUTATED=1 printf '\n// ab-selftest edit\n' >> "$MCP/$MCPREL" edited2=$(git -C "$MCP" hash-object "$MCPREL") ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$MCP/$MCPREL" >/tmp/ab_selftest6.out 2>&1 \ diff --git a/workspace/scripts/affected.ts b/workspace/scripts/affected.ts index ae6d916b..2410986e 100755 --- a/workspace/scripts/affected.ts +++ b/workspace/scripts/affected.ts @@ -162,7 +162,12 @@ async function main(): Promise { if (docsTokens.size > 0) { const ids = matchingEvalIds(evals, docsTokens); if (ids.length > 0) { - console.log(`mise run eval -- ${ids.map((id) => `--eval ${id}`).join(" ")}`); + // Docs impact is only measurable against the LOCAL index: the local mcp + // build (mcp-eval) pointed at the local content API. A bare `mise run + // eval` would query the production docs API and never see the edit. + console.log( + `SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql mise run mcp-eval -- ${ids.map((id) => `--eval ${id}`).join(" ")} # needs: mise run docs-index && mise run docs-api`, + ); commandCount++; } } @@ -175,13 +180,15 @@ async function main(): Promise { } } if (ids.size > 0) { - console.log(`mise run eval -- ${[...ids].sort().map((id) => `--eval ${id}`).join(" ")}`); + // mcp-eval builds submodules/mcp and selects it via SUPABASE_MCP_SERVER_PATH; + // a bare `mise run eval` would run the published npx server instead. + console.log(`mise run mcp-eval -- ${[...ids].sort().map((id) => `--eval ${id}`).join(" ")}`); commandCount++; } } if (serverWide) { - console.log("mise run eval -- --smoke"); + console.log("mise run mcp-eval -- --smoke"); commandCount++; } diff --git a/workspace/scripts/docs-embed-env.sh b/workspace/scripts/docs-embed-env.sh new file mode 100644 index 00000000..8eaf7b84 --- /dev/null +++ b/workspace/scripts/docs-embed-env.sh @@ -0,0 +1,22 @@ +# Shared preamble for the paid docs-embed paths (docs-index.sh, docs-seed.sh): +# fail-closed patch check, key loading, docs env, local-stack credentials, and +# the OpenAI preflight. Source from the repo root (the callers cd there first); +# not executable on its own. +if ! git -C supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then + echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 + exit 1 +fi + +source workspace/scripts/load-keys.sh +set -a +source supabase/apps/docs/.env.development +set +a +eval "$(supabase status --workdir supabase -o env)" +export NEXT_PUBLIC_SUPABASE_URL="$API_URL" +export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" +export SUPABASE_SECRET_KEY="$SECRET_KEY" +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" +workspace/scripts/openai-preflight.sh # fail fast if the key can't run embeddings +# Deliberately allow embedding without prod-only sources (e.g. lint-warnings, +# which needs DOCS_GITHUB_APP_*). Sources gate their own skip on this flag. +export DOCS_EMBED_ALLOW_MISSING_SOURCES=1 diff --git a/workspace/scripts/docs-index.sh b/workspace/scripts/docs-index.sh index fc68d07a..2ae557e6 100755 --- a/workspace/scripts/docs-index.sh +++ b/workspace/scripts/docs-index.sh @@ -2,24 +2,7 @@ set -euo pipefail cd "$(dirname "$0")/../.." -if ! git -C supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then - echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 - exit 1 -fi - -source workspace/scripts/load-keys.sh -set -a -source supabase/apps/docs/.env.development -set +a -eval "$(supabase status --workdir supabase -o env)" -export NEXT_PUBLIC_SUPABASE_URL="$API_URL" -export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" -export SUPABASE_SECRET_KEY="$SECRET_KEY" -: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" -workspace/scripts/openai-preflight.sh # fail fast if the key can't run embeddings -# Deliberately allow embedding without prod-only sources (e.g. lint-warnings, -# which needs DOCS_GITHUB_APP_*). Sources gate their own skip on this flag. -export DOCS_EMBED_ALLOW_MISSING_SOURCES=1 +source workspace/scripts/docs-embed-env.sh pnpm --dir supabase/apps/docs run embeddings node workspace/scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-seed.sh b/workspace/scripts/docs-seed.sh index 60aca009..3be06de4 100755 --- a/workspace/scripts/docs-seed.sh +++ b/workspace/scripts/docs-seed.sh @@ -2,24 +2,7 @@ set -euo pipefail cd "$(dirname "$0")/../.." -if ! git -C supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then - echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 - exit 1 -fi - -source workspace/scripts/load-keys.sh -set -a -source supabase/apps/docs/.env.development -set +a -eval "$(supabase status --workdir supabase -o env)" -export NEXT_PUBLIC_SUPABASE_URL="$API_URL" -export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" -export SUPABASE_SECRET_KEY="$SECRET_KEY" -: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" -workspace/scripts/openai-preflight.sh # fail fast if the key can't run embeddings -# Deliberately allow embedding without prod-only sources (e.g. lint-warnings, -# which needs DOCS_GITHUB_APP_*). Sources gate their own skip on this flag. -export DOCS_EMBED_ALLOW_MISSING_SOURCES=1 +source workspace/scripts/docs-embed-env.sh printf '%s\n' \ 'Full docs embedding rebuild' \ diff --git a/workspace/scripts/provenance.mjs b/workspace/scripts/provenance.mjs index 292f3c4f..93a05e16 100755 --- a/workspace/scripts/provenance.mjs +++ b/workspace/scripts/provenance.mjs @@ -77,15 +77,12 @@ const gitRaw = (dir, ...args) => { }; const sha256 = (data) => createHash("sha256").update(data).digest("hex"); -const repoState = (dir) => { - const dirty = (git(dir, "status", "--porcelain") ?? "").length > 0; - // Hash the TRACKED diff only when there is one: an untracked-only dirty - // tree keeps dirty:true but dirty_diff_sha256:null (the untracked list - // below carries that state) — a hash-of-empty-diff here would mislead. - const trackedDiff = dirty ? gitRaw(dir, "diff", "HEAD", "--binary") : null; - // Untracked files are invisible to `git diff HEAD` but are real workspace - // state (e.g. a brand-new guide page) — record path + content hash, sorted. - const untracked = (git(dir, "ls-files", "--others", "--exclude-standard") ?? "") +// Untracked files are invisible to `git diff HEAD` but are real workspace +// state (e.g. a brand-new guide page) — record path + content hash, sorted. +// One implementation for both the whole-repo receipt and the docs-scoped +// stamp, so the two can never fingerprint untracked state differently. +const hashUntracked = (dir, ...pathspec) => + (git(dir, "ls-files", "--others", "--exclude-standard", ...pathspec) ?? "") .split("\n") .filter(Boolean) .sort() @@ -96,6 +93,14 @@ const repoState = (dir) => { return { path: p, sha256: null }; } }); + +const repoState = (dir) => { + const dirty = (git(dir, "status", "--porcelain") ?? "").length > 0; + // Hash the TRACKED diff only when there is one: an untracked-only dirty + // tree keeps dirty:true but dirty_diff_sha256:null (the untracked list + // below carries that state) — a hash-of-empty-diff here would mislead. + const trackedDiff = dirty ? gitRaw(dir, "diff", "HEAD", "--binary") : null; + const untracked = hashUntracked(dir); return { branch: git(dir, "rev-parse", "--abbrev-ref", "HEAD"), sha: git(dir, "rev-parse", "HEAD"), @@ -222,18 +227,8 @@ if (cmd === "--stamp-docs-index") { } const dirtyDiff = gitRaw("supabase", "diff", "HEAD", "--binary", "--", DOCS_CONTENT_DIR); // Untracked corpus files (a brand-new guide) are part of the embedded state - // but invisible to `git diff HEAD` — record them path + content hash, sorted. - const contentUntracked = (git("supabase", "ls-files", "--others", "--exclude-standard", "--", DOCS_CONTENT_DIR) ?? "") - .split("\n") - .filter(Boolean) - .sort() - .map((p) => { - try { - return { path: p, sha256: sha256(readFileSync(join(root, "supabase", p))) }; - } catch { - return { path: p, sha256: null }; - } - }); + // but invisible to `git diff HEAD` — scoped to the docs content slice. + const contentUntracked = hashUntracked("supabase", "--", DOCS_CONTENT_DIR); const pipeline = embedPipelineConfig(); const stamp = { generated_at: new Date().toISOString(), From 514e75e454730d7366dbbfe42eb4fdf0ba0680a9 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 12:44:34 +0200 Subject: [PATCH 6/9] chore: team-review minors (all 11) - canary EVAL.ts imports @supabase-evals/core (real package; was elided at runtime but broke type resolution) - SUPABASE_CONTENT_API_URL env fallback now applies only alongside the SUPABASE_MCP_SERVER_PATH override: a stray env var can no longer feed --content-api-url to the published npx server (which rejects unknown flags). Explicit contentApiUrl option stays unconditional. Tests adjusted, +1 covering the stray-env npx case. - affected.ts: skips an eval dir missing PROMPT.md instead of aborting; the docs command's '# needs:' hint moved to its own line - content-API URL literals cross-referenced at each site (4-way sync note) - PUBLISH_REPOS identity alias collapsed into PATCH_REPOS everywhere - mise.toml pins pnpm 10.24.0 (matches packageManager/lockfile) - status.sh drops the stale 'incl. in agent-skills' guard comment - ab.sh fullwidth parens -> ASCII; .gitignore double blank dropped - status.test.sh anchors on the exact 'Ready. Try:' banner - hooks.test.sh/ab.test.sh fail actionably on zero checks instead of passing vacuously when submodules are missing Verified: status 19/0, hooks 8/0, ab 22/0, core mcp-server 12/12, framework typecheck clean, affected smoke on a real docs path. --- .gitignore | 1 - demo/canary-eval/EVAL.ts | 2 +- mise.toml | 2 +- packages/core/src/index.ts | 13 +++++++++---- packages/core/src/mcp-server.test.ts | 11 ++++++++++- workspace/scripts/ab.sh | 4 ++-- workspace/scripts/ab.test.sh | 7 ++++--- workspace/scripts/affected.ts | 5 ++++- workspace/scripts/apply-patches.sh | 2 +- workspace/scripts/hooks.test.sh | 6 +++++- workspace/scripts/patches-lib.sh | 9 +++------ workspace/scripts/publish.sh | 2 +- workspace/scripts/status.sh | 6 +++--- workspace/scripts/status.test.sh | 10 +++++----- 14 files changed, 49 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index 40a83456..2154efe1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ dist/ results/*/ .sync-tmp/ - # eval-source workspace glue (workspace/README.md) /supabase/ /results-ab/ diff --git a/demo/canary-eval/EVAL.ts b/demo/canary-eval/EVAL.ts index a0ca1162..0a8b1b12 100644 --- a/demo/canary-eval/EVAL.ts +++ b/demo/canary-eval/EVAL.ts @@ -1,4 +1,4 @@ -import { type CheckResult, type ToolScorer } from '@supabase/evals/core' +import { type CheckResult, type ToolScorer } from '@supabase-evals/core' // eval-workspace docs-discriminator demo: '@supabase/pinniped' is a fictional package // planted ONLY in the local docs by scripts/ab-demo.sh. It cannot come from model diff --git a/mise.toml b/mise.toml index 137c5666..ae940f03 100644 --- a/mise.toml +++ b/mise.toml @@ -6,7 +6,7 @@ [tools] node = "22" -pnpm = "10" +pnpm = "10.24.0" [tasks.setup] description = "Install deps, init submodules (agent-skills + mcp), apply patches, wire .env, print status" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f0ad75a5..44eb2ee0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -928,14 +928,19 @@ export function supabaseMcpServer( // platform-independent (it queries the public docs GraphQL API), so a // docs-only server runs standalone with no `--api-url`. if (apiUrl) serverArgs.push("--api-url", apiUrl); + + const local = resolveLocalMcpServer(); // Alternative docs Content API endpoint (e.g. a locally built docs - // index). Requires a server that understands --content-api-url — the - // SUPABASE_MCP_SERVER_PATH build; only set the env var alongside it. + // index). Only a server that understands --content-api-url can accept + // it (the SUPABASE_MCP_SERVER_PATH build; the published 0.8.1 npx + // package rejects unknown flags), so the ENV fallback applies only + // alongside the local override — a stray env var can't break a plain + // npx run. The explicit option is intentional and always honored. const contentApiUrl = - options.contentApiUrl ?? process.env.SUPABASE_CONTENT_API_URL; + options.contentApiUrl ?? + (local ? process.env.SUPABASE_CONTENT_API_URL : undefined); if (contentApiUrl) serverArgs.push("--content-api-url", contentApiUrl); - const local = resolveLocalMcpServer(); if (local) { // `node`, not process.execPath: CLI agents run this command INSIDE the // sandbox container, where the host's node binary path does not exist. diff --git a/packages/core/src/mcp-server.test.ts b/packages/core/src/mcp-server.test.ts index 5bfefb68..52056f01 100644 --- a/packages/core/src/mcp-server.test.ts +++ b/packages/core/src/mcp-server.test.ts @@ -46,8 +46,9 @@ describe("supabaseMcpServer().createConfig", () => { expect(config.args).not.toContain("--content-api-url"); }); - it("threads --content-api-url from the env var", async () => { + it("threads --content-api-url from the env var on the local override path", async () => { clearEnv(); + vi.stubEnv("SUPABASE_MCP_SERVER_PATH", fixtureDir); vi.stubEnv("SUPABASE_CONTENT_API_URL", "https://env.test/gql"); const { config } = await supabaseMcpServer().createConfig({}); const i = config.args.indexOf("--content-api-url"); @@ -55,6 +56,14 @@ describe("supabaseMcpServer().createConfig", () => { expect(config.args[i + 1]).toBe("https://env.test/gql"); }); + it("ignores a stray env var on the npx path (0.8.1 rejects unknown flags)", async () => { + clearEnv(); + vi.stubEnv("SUPABASE_CONTENT_API_URL", "https://env.test/gql"); + const { config } = await supabaseMcpServer().createConfig({}); + expect(config.command).toBe("npx"); + expect(config.args).not.toContain("--content-api-url"); + }); + it("prefers the explicit contentApiUrl option over the env var", async () => { clearEnv(); vi.stubEnv("SUPABASE_CONTENT_API_URL", "https://env.test/gql"); diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh index 462811b5..c7d6421e 100755 --- a/workspace/scripts/ab.sh +++ b/workspace/scripts/ab.sh @@ -51,7 +51,7 @@ for p in "${PATHS[@]}"; do done MCP="$PWD/submodules/mcp/packages/mcp-server-supabase" -CONTENT_URL="http://127.0.0.1:3001/docs/api/graphql" +CONTENT_URL="http://127.0.0.1:3001/docs/api/graphql" # also in affected.ts / docs-api.sh / workspace README — keep in sync RUN_ENV=() case "$LOOP" in docs) RUN_ENV=( "SUPABASE_MCP_SERVER_PATH=$MCP" "SUPABASE_CONTENT_API_URL=$CONTENT_URL" ) ;; @@ -70,7 +70,7 @@ sync() { if [ -n "${AB_DRYRUN:-}" ]; then echo "eval=$EVAL experiment=$EXP loop=$LOOP clone=$CLONE" echo "revert paths: ${REL[*]}" - echo "run env: ${RUN_ENV[*]:-（none）}" + echo "run env: ${RUN_ENV[*]:-(none)}" exit 0 fi diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh index 5cd33567..3de99070 100755 --- a/workspace/scripts/ab.test.sh +++ b/workspace/scripts/ab.test.sh @@ -7,20 +7,21 @@ # # SAFETY: only ever touches a tracked skill file that is currently CLEAN (no # working-tree or staged changes); it never checks out a file with your edits. -# If no clean skill file exists it SKIPS. Skills loop → sync is a no-op, so +# If no clean skill file exists it fails with an actionable message rather +# than passing vacuously. Skills loop → sync is a no-op, so # nothing else in the workspace is affected. Run: bash workspace/scripts/ab.test.sh set -euo pipefail cd "$(dirname "$0")/../.." SK=submodules/agent-skills -[ -e "$SK/.git" ] || { echo "SKIP: agent-skills submodule not initialized (run: mise run setup)"; exit 0; } +[ -e "$SK/.git" ] || { echo "ab.test: 0 checks ran — agent-skills submodule not initialized (run: mise run setup)" >&2; exit 1; } # pick a tracked SKILL.md with NO local changes, so restoring it can't lose work REL="" for c in $(git -C "$SK" ls-files 'skills/*/SKILL.md'); do if git -C "$SK" diff --quiet -- "$c" && git -C "$SK" diff --cached --quiet -- "$c"; then REL="$c"; break; fi done -[ -n "$REL" ] || { echo "SKIP: no CLEAN tracked skill file to test with"; exit 0; } +[ -n "$REL" ] || { echo "ab.test: 0 checks ran — no CLEAN tracked skill file to test with" >&2; exit 1; } FILE="$SK/$REL" EVAL=ab-selftest; EXP=claude-sonnet-5 RES="results/$EXP/$EVAL.json" diff --git a/workspace/scripts/affected.ts b/workspace/scripts/affected.ts index 2410986e..53bd9bb2 100755 --- a/workspace/scripts/affected.ts +++ b/workspace/scripts/affected.ts @@ -50,6 +50,7 @@ async function discoverEvals(): Promise { if (!statSync(evalDir).isDirectory()) continue; const promptPath = join(evalDir, "PROMPT.md"); + if (!existsSync(promptPath)) continue; // stray/partial dir: skip, don't abort the mapper const metadata = parseEvalMarkdown( readFileSync(promptPath, "utf8"), `evals/${id}/PROMPT.md`, @@ -165,8 +166,10 @@ async function main(): Promise { // Docs impact is only measurable against the LOCAL index: the local mcp // build (mcp-eval) pointed at the local content API. A bare `mise run // eval` would query the production docs API and never see the edit. + // URL also lives in ab.sh / docs-api.sh / workspace README — keep in sync. + console.log("# needs: mise run docs-index && mise run docs-api"); console.log( - `SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql mise run mcp-eval -- ${ids.map((id) => `--eval ${id}`).join(" ")} # needs: mise run docs-index && mise run docs-api`, + `SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql mise run mcp-eval -- ${ids.map((id) => `--eval ${id}`).join(" ")}`, ); commandCount++; } diff --git a/workspace/scripts/apply-patches.sh b/workspace/scripts/apply-patches.sh index 85f7a539..c44ed170 100755 --- a/workspace/scripts/apply-patches.sh +++ b/workspace/scripts/apply-patches.sh @@ -142,7 +142,7 @@ for repo in $PATCH_REPOS; do || { echo " ERROR: $dir index has staged changes — unstage first (git -C $dir status)" >&2; exit 1; } for p in $(patches_for "$repo"); do apply_one "$dir" "$p"; done done -for name in $PUBLISH_REPOS; do +for name in $PATCH_REPOS; do dir=$(repo_dir "$name") [ -e "$dir/.git" ] && install_pre_push_guard "$dir" done diff --git a/workspace/scripts/hooks.test.sh b/workspace/scripts/hooks.test.sh index d1fe81ef..4d6876ca 100755 --- a/workspace/scripts/hooks.test.sh +++ b/workspace/scripts/hooks.test.sh @@ -51,9 +51,13 @@ test_guard_in() { ck "$repo: guard active again" "$(grep -c 'eval-workspace pre-push guard' "$hook")" "1" } -for r in $PUBLISH_REPOS; do +for r in $PATCH_REPOS; do test_guard_in "$r" done echo "hooks.test: $pass passed, $fail failed" +if [ "$pass" -eq 0 ] && [ "$fail" -eq 0 ]; then + echo "hooks.test: 0 checks ran (no patched repo present — run: mise run setup)" >&2 + exit 1 +fi [ "$fail" -eq 0 ] diff --git a/workspace/scripts/patches-lib.sh b/workspace/scripts/patches-lib.sh index b2c5d700..0e491319 100644 --- a/workspace/scripts/patches-lib.sh +++ b/workspace/scripts/patches-lib.sh @@ -7,11 +7,9 @@ _MANIFEST_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) _manifest() { node "$_MANIFEST_LIB_DIR/manifest.mjs" "$@"; } -# PATCH_REPOS = repos carrying enabler patches (manifest order): mcp, supabase -# PUBLISH_REPOS = same set here — skills has no patches and is neither a patch -# nor a publish repo in this manifest (both derive from the -# same "has patches" predicate; kept as separate names because -# downstream scripts read them for different purposes) +# PATCH_REPOS = repos carrying enabler patches (manifest order): mcp, supabase. +# Patched repos are exactly the publishable ones today, so this is the one +# name; skills has no patches and is neither. Split only if the sets diverge. # # Fail-loud init: `for x in $(failing-cmd)` does NOT trip `set -e` in the # sourcing script — the substitution failure is swallowed and the lists come @@ -28,7 +26,6 @@ for _r in $_MANIFEST_REPOS; do PATCH_REPOS="${PATCH_REPOS:+$PATCH_REPOS }$_r" fi done -PUBLISH_REPOS="$PATCH_REPOS" unset _r _MANIFEST_REPOS repo_dir() { local d; d=$(_manifest get "$1" dir); echo "${d:-$1}"; } diff --git a/workspace/scripts/publish.sh b/workspace/scripts/publish.sh index 3b67a5c5..6cf58f0c 100755 --- a/workspace/scripts/publish.sh +++ b/workspace/scripts/publish.sh @@ -17,7 +17,7 @@ ROOT="$PWD" source workspace/scripts/patches-lib.sh repo="${1:-}"; shift || true -case " $PUBLISH_REPOS " in *" $repo "*) ;; *) +case " $PATCH_REPOS " in *" $repo "*) ;; *) echo "usage: mise run publish [--with ]... | --list" >&2; exit 2 ;; esac dir=$(repo_dir "$repo") diff --git a/workspace/scripts/status.sh b/workspace/scripts/status.sh index 247f9ee4..53d56f4b 100755 --- a/workspace/scripts/status.sh +++ b/workspace/scripts/status.sh @@ -29,7 +29,7 @@ echo "Repos:" repo_status "evals (host)" "." repo_status " - agent-skills" "$(repo_dir skills)" "not initialized" repo_status " - mcp" "$(repo_dir mcp)" "not initialized" -for _repo in $PUBLISH_REPOS; do +for _repo in $PATCH_REPOS; do case "$_repo" in skills|mcp) continue ;; esac repo_status "$_repo" "$(repo_dir "$_repo")" done @@ -55,8 +55,8 @@ for _r in $PATCH_REPOS; do fi done done -# pre-push guards active? (apply-patches installs them, incl. in agent-skills) -for _n in $PUBLISH_REPOS; do +# pre-push guards active? (apply-patches installs them) +for _n in $PATCH_REPOS; do _d=$(repo_dir "$_n") [ -e "$_d/.git" ] || continue _hooks=$(git -C "$_d" rev-parse --path-format=absolute --git-path hooks 2>/dev/null || true) diff --git a/workspace/scripts/status.test.sh b/workspace/scripts/status.test.sh index 93e5c616..18124cb0 100755 --- a/workspace/scripts/status.test.sh +++ b/workspace/scripts/status.test.sh @@ -32,14 +32,14 @@ run_case fresh state_fresh ck "fresh: suggests pnpm install" "$(has 'pnpm install')" y ck "fresh: suggests submodule init" "$(has 'submodule update --init')" y ck "fresh: suggests required keys" "$(has 'mise run store-key OPENAI_API_KEY')" y -ck "fresh: not Ready" "$(has 'Ready.')" n +ck "fresh: not Ready" "$(has 'Ready. Try:')" n # --- deps installed, submodules not yet initialized --- state_deps_only() { mkdir -p node_modules; cp .env.example .env; } run_case deps-only state_deps_only ck "deps-only: no pnpm-install suggestion" "$(has 'pnpm install')" n ck "deps-only: suggests submodule init" "$(has 'submodule update --init')" y -ck "deps-only: not Ready" "$(has 'Ready.')" n +ck "deps-only: not Ready" "$(has 'Ready. Try:')" n # --- submodules initialized (markers/hooks still missing), env missing --- state_submodules_init() { mkdir -p node_modules submodules/agent-skills/.git submodules/mcp/.git; } @@ -47,7 +47,7 @@ run_case submodules-init state_submodules_init ck "submodules-init: no pnpm-install suggestion" "$(has 'pnpm install')" n ck "submodules-init: no submodule-init suggestion" "$(has 'submodule update --init')" n ck "submodules-init: suggests required keys" "$(has 'mise run store-key ANTHROPIC_API_KEY')" y -ck "submodules-init: not Ready" "$(has 'Ready.')" n +ck "submodules-init: not Ready" "$(has 'Ready. Try:')" n # supabase was never cloned in this state — its absence must stay informational, # never surfacing an action item (there is no clone-docs step in the bootstrap order). ck "submodules-init: supabase absence isn't an action item" "$(has 'clone-docs')" n @@ -58,14 +58,14 @@ run_case keys-only state_keys_only ck "keys-only: no pnpm-install suggestion" "$(has 'pnpm install')" n ck "keys-only: no submodule-init suggestion" "$(has 'submodule update --init')" n ck "keys-only: suggests keys" "$(has 'mise run store-key ANTHROPIC_API_KEY')" y -ck "keys-only: not Ready" "$(has 'Ready.')" n +ck "keys-only: not Ready" "$(has 'Ready. Try:')" n # --- non-Darwin: same complete-but-keyless state; keys must route to .env, never store-key --- state_linux_keys() { state_keys_only; printf '#!/bin/sh\necho Linux\n' > bin/uname; } run_case linux-keys state_linux_keys ck "linux: no store-key suggestion" "$(has 'mise run store-key')" n ck "linux: routes keys to .env" "$(has 'add ANTHROPIC_API_KEY=')" y -ck "linux: not Ready" "$(has 'Ready.')" n +ck "linux: not Ready" "$(has 'Ready. Try:')" n echo "status.test: $pass passed, $fail failed" [ "$fail" -eq 0 ] From c5c4579a4e67aefd07504456912a4c055673e36f Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 12:51:13 +0200 Subject: [PATCH 7/9] refactor: one verification mechanism for patch-canonicality apply-patches verified 'state matches the canonical patch' two ways: byte-exact tree-OID reconstruction for an existing marker commit, but a grep-payload diff for a fresh build - the weaker check, blind to an identical +/- line reattributed across files within a multi-file patch. Extract patch_tree() (throwaway index: parent-tree + patch -> tree OID) and compare tree identity on BOTH paths; the payload-diff mechanism is deleted and its blind spot closes as a side effect. Verified live: existing-commit path ('already committed'), fresh path (marker dropped via rebase --onto, rebuilt through the new check); hooks 8/0, ab 22/0, status 19/0. --- workspace/scripts/apply-patches.sh | 34 ++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/workspace/scripts/apply-patches.sh b/workspace/scripts/apply-patches.sh index c44ed170..f64e5b7e 100755 --- a/workspace/scripts/apply-patches.sh +++ b/workspace/scripts/apply-patches.sh @@ -19,6 +19,21 @@ cd "$(dirname "$0")/../.." ROOT="$PWD" source workspace/scripts/patches-lib.sh +# Tree OID of " + ", built in a throwaway index. This is +# the ONE verification mechanism for "state matches the canonical patch": +# byte-exact tree identity, immune to the classic payload-diff blind spot +# (an identical +/- line reattributed to a different file). +patch_tree() { + local dir="$1" parent="$2" patch="$3" tmpidx out="" + tmpidx=$(mktemp /tmp/eval-workspace-idx-XXXXXX); rm -f "$tmpidx" # keep only the name: git must create the index itself + if GIT_INDEX_FILE="$tmpidx" git -C "$dir" read-tree "$parent" 2>/dev/null \ + && GIT_INDEX_FILE="$tmpidx" git -C "$dir" apply --cached "$ROOT/$patch" 2>/dev/null; then + out=$(GIT_INDEX_FILE="$tmpidx" git -C "$dir" write-tree 2>/dev/null || echo "") + fi + rm -f "$tmpidx" + echo "$out" +} + apply_one() { local dir="$1" patch="$2" subject sha h s subject=$(patch_subject "$patch") @@ -31,16 +46,10 @@ $(git -C "$dir" log --format='%H %s' HEAD --not --remotes 2>/dev/null) EOF if [ -n "$sha" ]; then # the .patch file is canonical — the commit must still match it EXACTLY: - # rebuild parent-tree + patch in a temp index and compare tree OIDs - local tmpidx want got - tmpidx=$(mktemp /tmp/eval-workspace-idx-XXXXXX); rm -f "$tmpidx" # keep only the name: git must create the index itself + # rebuild parent-tree + patch and compare tree OIDs + local want got want=$(git -C "$dir" rev-parse "$sha^{tree}") - got="" - if GIT_INDEX_FILE="$tmpidx" git -C "$dir" read-tree "$sha^" 2>/dev/null \ - && GIT_INDEX_FILE="$tmpidx" git -C "$dir" apply --cached "$ROOT/$patch" 2>/dev/null; then - got=$(GIT_INDEX_FILE="$tmpidx" git -C "$dir" write-tree 2>/dev/null || echo "") - fi - rm -f "$tmpidx" + got=$(patch_tree "$dir" "$sha^" "$patch") if [ "$got" = "$want" ]; then echo " $(basename "$patch") already committed in $dir" else @@ -61,11 +70,10 @@ EOF echo " ERROR: $(basename "$patch") does not apply cleanly to $dir — regenerate it (workspace/patches/README.md)" >&2 exit 1 fi - # the staged diff must be exactly the canonical patch (payload comparison) - if ! diff -q <(git -C "$dir" diff --cached | grep -E '^[+-]' | grep -vE '^(\+\+\+|---)') \ - <(grep -E '^[+-]' "$ROOT/$patch" | grep -vE '^(\+\+\+|---)') >/dev/null 2>&1; then + # the staged state must be exactly parent-tree + canonical patch (tree identity) + if [ "$(git -C "$dir" write-tree)" != "$(patch_tree "$dir" HEAD "$patch")" ]; then git -C "$dir" reset -q - echo " ERROR: staged diff for $(basename "$patch") differs from the canonical patch — aborted (index reset)" >&2 + echo " ERROR: staged state for $(basename "$patch") differs from the canonical patch — aborted (index reset)" >&2 exit 1 fi GIT_AUTHOR_NAME=eval-workspace GIT_AUTHOR_EMAIL=eval-workspace@local \ From 41a6efb22d97bc3c13947302ffeae7e8c1e0b77c Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 13:23:07 +0200 Subject: [PATCH 8/9] fix: two bugs surfaced by the cold-install A/B demo 1. mcp-build built the UNPATCHED server and orphaned the marker commit: it delegated to the root 'pnpm mcp:build', whose first step is another 'git submodule update --init' - on an initialized submodule that re-checks out the pin, discarding the marker apply-patches created a step earlier (and any user commits). The task now runs init -> install -> apply-patches -> build directly; the root script remains the evals-native unpatched flow. Verified: after mcp-build the marker is HEAD and the built stdio.js carries --content-api-url. 2. Arm receipts mislabeled the run: ab.sh passed RUN_ENV to the eval but not to the 'provenance --embed' step, so a local-build arm recorded mcp_override: null. The embed now runs under the same env; ab.test gains a receipt-records-override assertion (23/0). Found by a real cold-clone demo: GitHub clone -> setup -> unstaged mcp edit -> live A/B (baseline PASS 5/5 -> treatment FAIL 3/5, REGRESSED). --- mise.toml | 7 ++++++- workspace/scripts/ab.sh | 6 ++++-- workspace/scripts/ab.test.sh | 4 ++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mise.toml b/mise.toml index ae940f03..bf1dc244 100644 --- a/mise.toml +++ b/mise.toml @@ -64,7 +64,12 @@ run = "workspace/scripts/apply-patches.sh" [tasks.mcp-build] description = "Build the mcp submodule with the enabler patches applied" -run = "git submodule update --init submodules/mcp && workspace/scripts/apply-patches.sh && pnpm mcp:build" +# Order matters: `git submodule update` on an initialized submodule re-checks +# out the pin, ORPHANING the patch marker commit (and any user commits) — so +# init strictly BEFORE apply-patches, and never again after (which is why this +# does not delegate to the root `pnpm mcp:build`, whose first step is another +# submodule update; that script is the evals-native unpatched flow). +run = "git submodule update --init submodules/mcp && pnpm --dir submodules/mcp install && workspace/scripts/apply-patches.sh && pnpm --dir submodules/mcp build" [tasks.mcp-eval] description = "Run evals against the local mcp build (auto init+patch+build)" diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh index c7d6421e..f4229f72 100755 --- a/workspace/scripts/ab.sh +++ b/workspace/scripts/ab.sh @@ -108,8 +108,10 @@ run_eval() { # $1 = label [ -f "$RES" ] || { echo "no result at $RES — check the eval/experiment ids" >&2; exit 1; } cp "$RES" "$OUT/$EVAL.$1.json" # per-arm receipt: capture provenance AFTER this arm's sync, so baseline and - # treatment records differ by exactly the edit under test (report-only) - node workspace/scripts/provenance.mjs --embed "$OUT/$EVAL.$1.json" + # treatment records differ by exactly the edit under test (report-only). + # Same env as the run itself, or the receipt would claim mcp_override: null + # while the eval actually ran against the local build. + env ${RUN_ENV[@]+"${RUN_ENV[@]}"} node workspace/scripts/provenance.mjs --embed "$OUT/$EVAL.$1.json" } # Restoration is ONE idempotent path: pop the stash AND re-sync, so the index/ diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh index 3de99070..c1ad810f 100755 --- a/workspace/scripts/ab.test.sh +++ b/workspace/scripts/ab.test.sh @@ -107,6 +107,10 @@ if [ -e "$MCP/.git" ] && git -C "$MCP" diff --quiet -- "$MCPREL" && git -C "$MCP || { echo "FAIL: patch-owned A/B exited nonzero"; sed 's/^/ /' /tmp/ab_selftest6.out; fail=$((fail+1)); } ck "patch-owned: edit preserved" "$(git -C "$MCP" hash-object "$MCPREL")" "$edited2" ck "patch-owned: marker commit intact" "$(git -C "$MCP" rev-parse HEAD)" "$marker0" + # the arm receipt must record the override the run actually used (mcp loop + # sets SUPABASE_MCP_SERVER_PATH); a null here mislabels the arm + ck "patch-owned: receipt records mcp override" \ + "$(node -p 'try{require("./results-ab/ab-selftest.treatment.json").provenance.mcp_override.path?"y":"n"}catch{"n"}')" "y" AB_FAIL_BASELINE=1 ANTHROPIC_API_KEY=dummy AB_EVAL_CMD="$FAKE" AB_SYNC_CMD="$SYNC" bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$MCP/$MCPREL" >/dev/null 2>&1 || true ck "patch-owned: edit restored after failure" "$(git -C "$MCP" hash-object "$MCPREL")" "$edited2" ck "patch-owned: marker intact after failure" "$(git -C "$MCP" rev-parse HEAD)" "$marker0" From 25b5f73f8f9f37957c927bf18b93c78181bf536c Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 13:29:19 +0200 Subject: [PATCH 9/9] docs: mcp#343 merged upstream, not yet released - patch retires at the next release + pin bump --- workspace/patches/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workspace/patches/README.md b/workspace/patches/README.md index 420778fe..0b54a8c5 100644 --- a/workspace/patches/README.md +++ b/workspace/patches/README.md @@ -26,7 +26,7 @@ derive from the kind). | Patch | Repo | Files | Kind | Tests | What | |---|---|---|---|---|---| -| `mcp-content-api-url` | `submodules/mcp` | `transports/stdio.ts` | upstream | 3 stdio integration tests (in the PR) | `--content-api-url` flag + `SUPABASE_CONTENT_API_URL` — **PR open: [mcp#343](https://github.com/supabase/mcp/pull/343)**; the patch retires when it merges and the pin moves past it | +| `mcp-content-api-url` | `submodules/mcp` | `transports/stdio.ts` | upstream | 3 stdio integration tests (upstream) | `--content-api-url` flag + `SUPABASE_CONTENT_API_URL` — **merged upstream ([mcp#343](https://github.com/supabase/mcp/pull/343), 2026-07-23), not yet in a release**; the patch retires when a release ships the flag and the pin moves past it | | `supabase-content-local-ports` | supabase/supabase | `config.toml` | **LOCAL-ONLY** | n/a | dev ports 55321+ (avoid evals' 54321-9) | | `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report | | `supabase-docs-guide-checksum` | supabase/supabase | `guideModelLoader.ts` | upstream | none | guide checksum + `tryCatch` `onError` fix (docs-index edit detection) | @@ -50,7 +50,7 @@ verifies both directions and fails loudly on drift. ## Upstreaming order -`mcp-content-api-url` — PR open ([mcp#343](https://github.com/supabase/mcp/pull/343)). +`mcp-content-api-url` — merged upstream ([mcp#343](https://github.com/supabase/mcp/pull/343)); retires at the next mcp release + pin bump. Then the three upstream supabase fixes (independent; `guide-checksum` and `index-fail-closed` still need regression tests; `reference-dup-sources` is test-covered and PR-ready). The two former evals patches are done: the