From 7694f7f5c2f27cabd7bbf7f3ef6fc0a5d529eb2a Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 13:53:05 +0200 Subject: [PATCH 01/18] M3: docs monorepo as a sparse+partial submodule (submodules/supabase) Replaces the managed clone-docs flat clone with a pinned submodule, so all three source repos (agent-skills, mcp, supabase) share one mechanism and the docs pin gives in-repo corpus provenance. - Seed recipe (verified: ~56s, 311M .git + 622M worktree vs 2.9G flat): blob-filtered clone without checkout, cone sparse-checkout, explicit checkout of the evals gitlink pin. Re-runs refresh sparse only. - Guardrails: 'update = none' (a recursive submodule update would materialize the ENTIRE monorepo tree - verified: 'git submodule update --init' and CI's recursive checkout skip it; the seed owns the checkout) and 'ignore = all' (marker commits and pin-bump rebases never show as dirt in evals). - Pin bumps: update.sh keeps the fetch+rebase-markers flow and now prints the deliberate-record step (git add submodules/supabase). The generic managed-clone flow is deleted - nothing used it anymore. - Pinned at 846706b, the base the current local index was embedded from (zero re-embed drift; the shared content DB carries over as-is). - status.sh: supabase rendered as a third nested submodule row + a git version floor check (seed needs partial clone + cone sparse, git 2.26+). - pnpm split: the docs monorepo pins its own pnpm (packageManager 11.13.1) vs evals' mise-pinned 10.24 - all docs-loop pnpm calls go through 'corepack pnpm' with cwd inside the submodule so the monorepo wins. - docs-content-api.ts / docs-api.sh: submodule path is one level deeper; fixed the route import and tsx entry path depth. Verified: seed + apply-patches (5 markers) + pre-push guards; trap check; self-tests 19/16/23 green; docs-api serves searchDocs from the seeded index (1769 pages) through the submodule route. --- .gitignore | 1 - .gitmodules | 5 +++ mise.toml | 4 +- submodules/supabase | 1 + workspace/README.md | 13 +++--- workspace/manifest.json | 8 ++-- workspace/patches/README.md | 2 +- workspace/scripts/ab-demo.sh | 8 ++-- workspace/scripts/ab-ready.sh | 4 +- workspace/scripts/ab.sh | 8 ++-- workspace/scripts/apply-patches.sh | 4 +- workspace/scripts/clone-docs.sh | 46 +++++++++++++++------ workspace/scripts/docs-api.sh | 17 ++++++-- workspace/scripts/docs-content-api.ts | 2 +- workspace/scripts/docs-down.sh | 2 +- workspace/scripts/docs-embed-env.sh | 6 +-- workspace/scripts/docs-index.sh | 4 +- workspace/scripts/docs-seed.sh | 2 +- workspace/scripts/docs-up.sh | 2 +- workspace/scripts/provenance.mjs | 12 +++--- workspace/scripts/publish.sh | 2 +- workspace/scripts/status.sh | 27 +++++++------ workspace/scripts/update.sh | 57 ++++++++++++++------------- 23 files changed, 142 insertions(+), 95 deletions(-) create mode 160000 submodules/supabase diff --git a/.gitignore b/.gitignore index 2154efe1..20ebb2ca 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ results/*/ .sync-tmp/ # eval-source workspace glue (workspace/README.md) -/supabase/ /results-ab/ /.docs-index-stamp.json /.publish/ diff --git a/.gitmodules b/.gitmodules index f3863093..1bebb112 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,8 @@ path = submodules/mcp url = git@github.com:supabase/mcp.git ignore = all +[submodule "submodules/supabase"] + path = submodules/supabase + url = git@github.com:supabase/supabase.git + ignore = all + update = none diff --git a/mise.toml b/mise.toml index bf1dc244..37ff8f10 100644 --- a/mise.toml +++ b/mise.toml @@ -21,7 +21,7 @@ description = "Run evals with the workspace env applied (args pass to `pnpm eval run = "workspace/scripts/eval.sh" [tasks.update] -description = "Fetch + fast-forward the supabase clone, re-applying patches (-- --check = report only)" +description = "Update the docs submodule: fetch + rebase local work onto upstream (-- --check = report only)" run = "workspace/scripts/update.sh" [tasks.publish] @@ -79,7 +79,7 @@ run = "workspace/scripts/mcp-eval.sh" # --- Docs loop (optional, heavy — needs Docker + the supabase CLI) --- [tasks.clone-docs] -description = "Sparse-clone supabase/supabase (apps/docs + deps) and install. Run once" +description = "Seed the sparse supabase docs submodule (apps/docs + deps) and install. Run once" run = "workspace/scripts/clone-docs.sh" [tasks.docs-up] diff --git a/submodules/supabase b/submodules/supabase new file mode 160000 index 00000000..db16c899 --- /dev/null +++ b/submodules/supabase @@ -0,0 +1 @@ +Subproject commit db16c89986979822e41bc3b0f0292dc7f52d90eb diff --git a/workspace/README.md b/workspace/README.md index bae2074f..6406b56f 100644 --- a/workspace/README.md +++ b/workspace/README.md @@ -31,7 +31,7 @@ put keys in the repo-root `.env` (the fallback `status` will route you to). |---|---|---| | Skills | `submodules/agent-skills` (a working tree: edit in place) | none | | MCP server | `submodules/mcp` | `mise run mcp-build` | -| Docs | `supabase/` (opt-in sparse clone: `mise run clone-docs`) | `mise run docs-index` (cents) | +| Docs | `submodules/supabase` (opt-in sparse+partial submodule: `mise run clone-docs`) | `mise run docs-index` (cents) | - **MCP loop**: `mise run mcp-eval -- ` builds the submodule (with the enabler patches) and runs evals against it via `SUPABASE_MCP_SERVER_PATH`. @@ -44,7 +44,10 @@ put keys in the repo-root `.env` (the fallback `status` will route you to). (or just use `mise run ab`, which wires this automatically). Content DB ports are 55321+ to avoid the eval local-stack range (54321-9). Measuring docs impact needs a tools-mode (`interface: mcp`) eval whose answer lives - only in the docs. + only in the docs. The submodule is pinned by evals (`ignore = all`, + `update = none`: recursive inits skip it — the seed script owns the sparse + checkout); bump the pin with `mise run update`, then `git add + submodules/supabase` deliberately. ## Head-to-head A/B @@ -62,8 +65,8 @@ self-cleaning live proof on the docs loop (spend-gated, asks first). ## Patches & publishing -Local changes to the patched repos (the `supabase/` clone and the -`submodules/mcp` working tree) are tracked as `.patch` files in +Local changes to the patched repos (the `submodules/supabase` and +`submodules/mcp` working trees) are tracked as `.patch` files in `workspace/patches/` and applied as marker commits — see [patches/README.md](./patches/README.md) for the manifest and the publish flow (`mise run publish `). A pre-push guard in each @@ -73,7 +76,7 @@ never gets hooks or marker commits. ## Provenance `mise run status -- --json` prints a receipt: host repo SHA + dirty state, -submodule pins, supabase clone state, patch fingerprints, and the docs-index +submodule pins, docs submodule state, patch fingerprints, and the docs-index stamp (`.docs-index-stamp.json`, scoped to repo docs content only). `ab.sh` embeds a per-arm copy into every A/B result, so a wrong-baseline run is immediately obvious. diff --git a/workspace/manifest.json b/workspace/manifest.json index 38858b1d..77b06200 100644 --- a/workspace/manifest.json +++ b/workspace/manifest.json @@ -3,11 +3,13 @@ "mcp": { "dir": "submodules/mcp", "kind": "submodule", - "patches": ["mcp-content-api-url"] + "patches": [ + "mcp-content-api-url" + ] }, "supabase": { - "dir": "supabase", - "remote": "git@github.com:supabase/supabase.git", + "dir": "submodules/supabase", + "kind": "submodule", "patches": [ "supabase-content-local-ports", "supabase-docs-index-fail-closed", diff --git a/workspace/patches/README.md b/workspace/patches/README.md index 0b54a8c5..b43680d3 100644 --- a/workspace/patches/README.md +++ b/workspace/patches/README.md @@ -1,6 +1,6 @@ # Enabler patches -Local changes to the patched repos (the `supabase/` clone and the +Local changes to the patched repos (the `submodules/supabase` and the `submodules/mcp` working tree), materialized by `workspace/scripts/apply-patches.sh` as identifiable **local commits** at the bottom of each repo's branch: diff --git a/workspace/scripts/ab-demo.sh b/workspace/scripts/ab-demo.sh index df73f3ee..1c9a7dcd 100755 --- a/workspace/scripts/ab-demo.sh +++ b/workspace/scripts/ab-demo.sh @@ -16,7 +16,7 @@ set -euo pipefail cd "$(dirname "$0")/../.." -GUIDE=supabase/apps/docs/content/guides/auth/choosing-a-server-package.mdx +GUIDE=submodules/supabase/apps/docs/content/guides/auth/choosing-a-server-package.mdx EVAL_ID=investigate-workspace-canary-nimbus-package EVAL_DST=evals/$EVAL_ID @@ -29,14 +29,14 @@ for k in ANTHROPIC_API_KEY OPENAI_API_KEY; do done # --- hard preflight (same gates the A/B itself needs) --- -[ -e supabase/.git ] || fail "supabase not cloned" +[ -e submodules/supabase/.git ] || fail "supabase not cloned" [ -n "${ANTHROPIC_API_KEY:-}" ] || fail "ANTHROPIC_API_KEY missing" [ -n "${OPENAI_API_KEY:-}" ] || fail "OPENAI_API_KEY missing" docker exec supabase_db_eval-workspace-content true 2>/dev/null || fail "content DB not running" pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) [ "${pages:-0}" -gt 0 ] 2>/dev/null || fail "docs index not seeded" curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql || fail "docs-api not serving on :3001" -git -C supabase diff --quiet -- "${GUIDE#supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}" +git -C submodules/supabase diff --quiet -- "${GUIDE#supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}" [ ! -e "$EVAL_DST" ] || fail "$EVAL_DST already exists — remove it first" echo "This runs a live A/B: 2 claude-sonnet-5 runs + a few embedding cents, ~3-5 min." @@ -46,7 +46,7 @@ read -r -p "Type 'demo' to proceed: " confirmation # --- plant the canary (append-only; reverted by the cleanup trap) --- cleanup() { echo "== demo cleanup: reverting canary, de-embedding, removing throwaway eval ==" - git -C supabase checkout -- "${GUIDE#supabase/}" 2>/dev/null || true + git -C submodules/supabase checkout -- "${GUIDE#supabase/}" 2>/dev/null || true rm -rf "$EVAL_DST" workspace/scripts/docs-index.sh >/dev/null 2>&1 \ || echo "WARNING: cleanup re-embed failed — run \`mise run docs-index\` to de-embed the canary" >&2 diff --git a/workspace/scripts/ab-ready.sh b/workspace/scripts/ab-ready.sh index 9b375f68..5830865a 100755 --- a/workspace/scripts/ab-ready.sh +++ b/workspace/scripts/ab-ready.sh @@ -26,8 +26,8 @@ else fi echo -echo "docs loop (edit supabase/apps/docs/content/… pages):" -if [ -e supabase/.git ]; then ok "supabase (apps/docs) cloned"; else miss "supabase not cloned" "mise run clone-docs"; fi +echo "docs loop (edit submodules/supabase/apps/docs/content/… pages):" +if [ -e submodules/supabase/.git ]; then ok "supabase (apps/docs) cloned"; else miss "supabase not cloned" "mise run clone-docs"; fi if docker exec supabase_db_eval-workspace-content true 2>/dev/null; then ok "content DB running" pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh index f4229f72..ed14dc71 100755 --- a/workspace/scripts/ab.sh +++ b/workspace/scripts/ab.sh @@ -5,7 +5,7 @@ # Usage: workspace/scripts/ab.sh [more-paths...] # is a file inside a clone; its clone selects the loop + how to # re-sync between states: -# supabase/apps/docs/content/… docs loop (re-embed via docs-index; needs `mise run docs-api` up) +# submodules/supabase/apps/docs/content/… docs loop (re-embed via docs-index; needs `mise run docs-api` up) # submodules/mcp/… mcp loop (rebuild the local server) # submodules/agent-skills/… skills loop (no re-sync; read live via symlink) # @@ -34,11 +34,11 @@ done # --- clone + loop from the first path --- case "${PATHS[0]}" in - supabase/apps/docs/content/*) LOOP=docs; CLONE=supabase; STRIP=supabase/; PREFIX=supabase/apps/docs/content/ ;; - supabase/apps/docs/*) echo "docs A/B works on content pages (supabase/apps/docs/content/…) — other docs files aren't part of the embed loop: ${PATHS[0]}" >&2; exit 2 ;; + submodules/supabase/apps/docs/content/*) LOOP=docs; CLONE=submodules/supabase; STRIP=submodules/supabase/; PREFIX=submodules/supabase/apps/docs/content/ ;; + submodules/supabase/apps/docs/*) echo "docs A/B works on content pages (submodules/supabase/apps/docs/content/…) — other docs files aren't part of the embed loop: ${PATHS[0]}" >&2; exit 2 ;; submodules/mcp/*) LOOP=mcp; CLONE=submodules/mcp; STRIP=submodules/mcp/; PREFIX=submodules/mcp/ ;; submodules/agent-skills/*) LOOP=skills; CLONE=submodules/agent-skills; STRIP=submodules/agent-skills/; PREFIX=submodules/agent-skills/ ;; - *) echo "path must be under supabase/apps/docs/content/, submodules/mcp/, or submodules/agent-skills/: ${PATHS[0]}" >&2; exit 2 ;; + *) echo "path must be under submodules/supabase/apps/docs/content/, submodules/mcp/, or submodules/agent-skills/: ${PATHS[0]}" >&2; exit 2 ;; esac # every path must live in this loop's editable scope; build clone-relative paths diff --git a/workspace/scripts/apply-patches.sh b/workspace/scripts/apply-patches.sh index f64e5b7e..e7f6e832 100755 --- a/workspace/scripts/apply-patches.sh +++ b/workspace/scripts/apply-patches.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Materialize the enabler patches as identifiable LOCAL COMMITS in the -# supabase clone and the mcp submodule working tree: +# supabase and mcp submodule working trees: # [eval-workspace-local] dev shim — must never leave this machine # [eval-workspace-upstream] upstream candidate — leaves ONLY via # `mise run publish … --with ` (reworded) @@ -9,7 +9,7 @@ # # Idempotent: a patch whose marker commit already exists is skipped; patch # content found uncommitted in the working tree (the old model) is migrated -# into a commit. Also installs a pre-push guard in the supabase clone and +# into a commit. Also installs a pre-push guard in the supabase submodule and # the mcp submodule (the two publishable repos) that blocks marker commits # from being pushed anywhere. A repo whose dir is absent (supabase not # cloned) or whose submodule is uninitialized (empty working tree, no .git) diff --git a/workspace/scripts/clone-docs.sh b/workspace/scripts/clone-docs.sh index 70b2f6fd..ce13bb53 100755 --- a/workspace/scripts/clone-docs.sh +++ b/workspace/scripts/clone-docs.sh @@ -1,21 +1,33 @@ #!/usr/bin/env bash +# Seed the docs submodule (submodules/supabase): a sparse+partial checkout of +# the supabase monorepo (apps/docs + the packages it imports), pinned by the +# evals gitlink. ~90s / ~1GB instead of the multi-GB full monorepo. +# +# The submodule is deliberately `update = none` in .gitmodules: a recursive +# `git submodule update --init` (or a --recurse-submodules clone) would +# materialize the ENTIRE monorepo tree — sparse-checkout cannot be injected +# before the atomic clone+checkout. So this script owns the seed: clone +# without checkout, configure sparse, then an explicit checkout of the pin. +# +# Re-runs refresh the sparse path set only; they never move HEAD (your marker +# commits and local work stay put — pin bumps go through `mise run update`). set -euo pipefail - cd "$(dirname "$0")/../.." -source workspace/scripts/patches-lib.sh -SUPABASE_REMOTE="${SUPABASE_REMOTE:-$(repo_remote supabase)}" -if [ -e supabase/.git ]; then - echo "supabase/ already cloned; refreshing sparse checkout" -elif [ -e supabase ]; then - echo "ERROR: supabase/ exists but is not a Git clone" >&2 - exit 1 +SUB=submodules/supabase +URL=${SUPABASE_REMOTE:-$(git config -f .gitmodules "submodule.$SUB.url")} + +FRESH=0 +if [ -e "$SUB/.git" ]; then + echo "$SUB already seeded; refreshing sparse checkout" else - git clone --filter=blob:none --no-checkout "$SUPABASE_REMOTE" supabase - git -C supabase sparse-checkout init --cone + FRESH=1 + # the gitlink placeholder is an empty dir on fresh checkouts; clone into it + git clone --filter=blob:none --no-checkout "$URL" "$SUB" + git -C "$SUB" sparse-checkout init --cone fi -git -C supabase sparse-checkout set \ +git -C "$SUB" sparse-checkout set \ apps/docs \ examples \ packages/ai-commands packages/api-types packages/build-icons \ @@ -24,5 +36,13 @@ git -C supabase sparse-checkout set \ packages/tsconfig packages/ui packages/ui-patterns \ patches supabase -git -C supabase checkout -pnpm --dir supabase install --filter docs... --frozen-lockfile +if [ "$FRESH" = 1 ]; then + # check out exactly the pinned SHA (blob filter fetches content on demand) + WANT=$(git rev-parse "HEAD:$SUB") + git -C "$SUB" cat-file -e "$WANT^{commit}" 2>/dev/null || git -C "$SUB" fetch -q origin "$WANT" + git -C "$SUB" checkout -q "$WANT" + echo "seeded $SUB at $(git -C "$SUB" rev-parse --short HEAD) (the evals pin)" +fi + +# corepack: the monorepo pins its own pnpm (packageManager) — see docs-api.sh +(cd "$SUB" && COREPACK_ENABLE_DOWNLOAD_PROMPT=0 corepack pnpm install --filter docs... --frozen-lockfile) diff --git a/workspace/scripts/docs-api.sh b/workspace/scripts/docs-api.sh index aa1e82c1..e94d3f2c 100755 --- a/workspace/scripts/docs-api.sh +++ b/workspace/scripts/docs-api.sh @@ -6,15 +6,24 @@ cd "$(dirname "$0")/../.." source workspace/scripts/load-keys.sh set -a -source supabase/apps/docs/.env.development +source submodules/supabase/apps/docs/.env.development set +a -eval "$(supabase status --workdir supabase -o env)" +eval "$(supabase status --workdir submodules/supabase -o env)" : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" +# corepack, cwd inside the submodule: the docs monorepo pins its OWN pnpm +# (packageManager, 11.x) which differs from evals' mise-pinned pnpm — corepack +# resolves the nearest packageManager upward from cwd. Entry path is relative +# to apps/docs (4 levels up to the evals root). The Sentry stub hook no-ops +# the route handler's telemetry calls (see sentry-stub.mjs). +STUB_REGISTER=$PWD/workspace/scripts/sentry-stub-register.mjs +cd submodules/supabase/apps/docs NODE_ENV=development \ NEXT_PUBLIC_SUPABASE_URL="$API_URL" \ NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" \ OPENAI_API_KEY="$OPENAI_API_KEY" \ -exec pnpm --dir supabase/apps/docs exec tsx \ +COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ +NODE_OPTIONS="--import $STUB_REGISTER${NODE_OPTIONS:+ $NODE_OPTIONS}" \ +exec corepack pnpm exec tsx \ --conditions=react-server \ --tsconfig tsconfig.json \ - ../../../workspace/scripts/docs-content-api.ts + ../../../../workspace/scripts/docs-content-api.ts diff --git a/workspace/scripts/docs-content-api.ts b/workspace/scripts/docs-content-api.ts index e869d938..ef5dd017 100644 --- a/workspace/scripts/docs-content-api.ts +++ b/workspace/scripts/docs-content-api.ts @@ -1,5 +1,5 @@ import { createServer } from 'node:http' -import { GET, OPTIONS, POST } from '../../supabase/apps/docs/app/api/graphql/route.ts' +import { GET, OPTIONS, POST } from '../../submodules/supabase/apps/docs/app/api/graphql/route.ts' const handlers = { GET, OPTIONS, POST } const port = Number(process.env.PORT ?? 3001) diff --git a/workspace/scripts/docs-down.sh b/workspace/scripts/docs-down.sh index 182dd86b..2590b9e4 100755 --- a/workspace/scripts/docs-down.sh +++ b/workspace/scripts/docs-down.sh @@ -2,4 +2,4 @@ set -euo pipefail cd "$(dirname "$0")/../.." -supabase stop --workdir supabase +supabase stop --workdir submodules/supabase diff --git a/workspace/scripts/docs-embed-env.sh b/workspace/scripts/docs-embed-env.sh index 8eaf7b84..f4071bf9 100644 --- a/workspace/scripts/docs-embed-env.sh +++ b/workspace/scripts/docs-embed-env.sh @@ -2,16 +2,16 @@ # fail-closed patch check, key loading, docs env, local-stack credentials, and # the OpenAI preflight. Source from the repo root (the callers cd there first); # not executable on its own. -if ! git -C supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then +if ! git -C submodules/supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 exit 1 fi source workspace/scripts/load-keys.sh set -a -source supabase/apps/docs/.env.development +source submodules/supabase/apps/docs/.env.development set +a -eval "$(supabase status --workdir supabase -o env)" +eval "$(supabase status --workdir submodules/supabase -o env)" export NEXT_PUBLIC_SUPABASE_URL="$API_URL" export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" export SUPABASE_SECRET_KEY="$SECRET_KEY" diff --git a/workspace/scripts/docs-index.sh b/workspace/scripts/docs-index.sh index 2ae557e6..a3e2fced 100755 --- a/workspace/scripts/docs-index.sh +++ b/workspace/scripts/docs-index.sh @@ -4,5 +4,7 @@ set -euo pipefail cd "$(dirname "$0")/../.." source workspace/scripts/docs-embed-env.sh -pnpm --dir supabase/apps/docs run embeddings +# corepack: the docs monorepo pins its own pnpm (packageManager), newer than +# evals' mise-pinned pnpm — see docs-api.sh +(cd submodules/supabase/apps/docs && COREPACK_ENABLE_DOWNLOAD_PROMPT=0 corepack pnpm run embeddings) node workspace/scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-seed.sh b/workspace/scripts/docs-seed.sh index 3be06de4..0c4a4060 100755 --- a/workspace/scripts/docs-seed.sh +++ b/workspace/scripts/docs-seed.sh @@ -14,5 +14,5 @@ if [ "$confirmation" != seed ]; then exit 1 fi -pnpm --dir supabase/apps/docs run embeddings:refresh +(cd submodules/supabase/apps/docs && COREPACK_ENABLE_DOWNLOAD_PROMPT=0 corepack pnpm run embeddings:refresh) node workspace/scripts/provenance.mjs --stamp-docs-index diff --git a/workspace/scripts/docs-up.sh b/workspace/scripts/docs-up.sh index 1971d65f..87f270c7 100755 --- a/workspace/scripts/docs-up.sh +++ b/workspace/scripts/docs-up.sh @@ -2,7 +2,7 @@ set -euo pipefail cd "$(dirname "$0")/../.." -supabase start --workdir supabase \ +supabase start --workdir submodules/supabase \ -x realtime,storage-api,imgproxy,mailpit,postgres-meta,studio,edge-runtime,logflare,vector,supavisor # Upstream page migrations grant service_role only Dxt (no CRUD). The new secret diff --git a/workspace/scripts/provenance.mjs b/workspace/scripts/provenance.mjs index 93a05e16..93f42820 100755 --- a/workspace/scripts/provenance.mjs +++ b/workspace/scripts/provenance.mjs @@ -44,7 +44,7 @@ const DOCS_CONTENT_DIR = "apps/docs/content"; // The embed pipeline's model/dims are read FROM the pipeline source at stamp // time (no duplicated literals that can silently drift); extraction failure // fails the stamp closed. -const EMBED_PIPELINE_FILE = "supabase/apps/docs/scripts/search/generate-embeddings.ts"; +const EMBED_PIPELINE_FILE = "submodules/supabase/apps/docs/scripts/search/generate-embeddings.ts"; const embedPipelineConfig = () => { const src = readFileSync(join(root, EMBED_PIPELINE_FILE), "utf8"); const model = src.match(/EMBEDDING_MODEL:\s*'([^']+)'/)?.[1]; @@ -222,13 +222,13 @@ if (cmd === "--stamp-docs-index") { // Fail closed on CONTENT (write no stamp, remove any stale one so nothing // misdescribes the new embed) but exit 0 with a loud warning. try { - if (!existsSync(join(root, "supabase", ".git"))) { + if (!existsSync(join(root, "submodules/supabase", ".git"))) { throw new Error("--stamp-docs-index needs the supabase clone"); } - const dirtyDiff = gitRaw("supabase", "diff", "HEAD", "--binary", "--", DOCS_CONTENT_DIR); + const dirtyDiff = gitRaw("submodules/supabase", "diff", "HEAD", "--binary", "--", DOCS_CONTENT_DIR); // Untracked corpus files (a brand-new guide) are part of the embedded state // but invisible to `git diff HEAD` — scoped to the docs content slice. - const contentUntracked = hashUntracked("supabase", "--", DOCS_CONTENT_DIR); + const contentUntracked = hashUntracked("submodules/supabase", "--", DOCS_CONTENT_DIR); const pipeline = embedPipelineConfig(); const stamp = { generated_at: new Date().toISOString(), @@ -241,9 +241,9 @@ if (cmd === "--stamp-docs-index") { // needs pipeline-side per-page checksum provenance (upstream fix first). scope: "repo-docs-content-only", external_sources_not_captured: true, - supabase_sha: git("supabase", "rev-parse", "HEAD"), + supabase_sha: git("submodules/supabase", "rev-parse", "HEAD"), repo_docs_content_state: { - content_tree: git("supabase", "rev-parse", `HEAD:${DOCS_CONTENT_DIR}`), + content_tree: git("submodules/supabase", "rev-parse", `HEAD:${DOCS_CONTENT_DIR}`), content_dirty_diff_sha256: dirtyDiff && dirtyDiff.length > 0 ? sha256(dirtyDiff) : null, content_untracked: contentUntracked, }, diff --git a/workspace/scripts/publish.sh b/workspace/scripts/publish.sh index 6cf58f0c..dcc7d7df 100755 --- a/workspace/scripts/publish.sh +++ b/workspace/scripts/publish.sh @@ -10,7 +10,7 @@ # Usage: # workspace/scripts/publish.sh --list what's publishable # workspace/scripts/publish.sh [--with ]... -# : mcp (submodule) | supabase (clone) — the only publishable repos +# : mcp (submodule) | supabase (docs submodule) — the only publishable repos set -euo pipefail cd "$(dirname "$0")/../.." ROOT="$PWD" diff --git a/workspace/scripts/status.sh b/workspace/scripts/status.sh index 53d56f4b..0d993f98 100755 --- a/workspace/scripts/status.sh +++ b/workspace/scripts/status.sh @@ -24,26 +24,24 @@ repo_status() { NEXT="" echo "Repos:" -# The host repo is this checkout itself (evals); agent-skills and mcp are its -# two pinned submodules (hand-nested rows); supabase is the only opt-in clone. +# The host repo is this checkout itself (evals); agent-skills, mcp, and +# supabase (docs) are its pinned submodules (hand-nested rows). supabase is +# opt-in: `update = none` keeps recursive inits away from the monorepo tree, +# so it only exists after `mise run clone-docs` seeds it (sparse+partial). repo_status "evals (host)" "." repo_status " - agent-skills" "$(repo_dir skills)" "not initialized" repo_status " - mcp" "$(repo_dir mcp)" "not initialized" -for _repo in $PATCH_REPOS; do - case "$_repo" in skills|mcp) continue ;; esac - repo_status "$_repo" "$(repo_dir "$_repo")" -done -unset _repo +repo_status " - supabase (docs)" "$(repo_dir supabase)" "not seeded" # Bootstrap order (mirrors mise run setup): deps, then submodules, then -# enabler plumbing, then keys. The supabase clone is opt-in for the docs loop -# only — its absence is informational (shown above) and never gates Ready. +# enabler plumbing, then keys. The supabase docs submodule is opt-in for the +# docs loop only — its absence is informational (shown above) and never gates Ready. [ -d node_modules ] || NEXT="${NEXT} pnpm install # install workspace deps\n" if [ ! -e "$(repo_dir skills)/.git" ] || [ ! -e "$(repo_dir mcp)/.git" ]; then NEXT="${NEXT} git submodule update --init submodules/agent-skills submodules/mcp # init submodules\n" fi # enabler plumbing present? (marker commits; see apply-patches.sh) — only -# checked for repos that are actually present (supabase clone, mcp submodule). +# checked for repos that are actually present (supabase and mcp submodules). for _r in $PATCH_REPOS; do _d=$(repo_dir "$_r") [ -e "$_d/.git" ] || continue @@ -85,9 +83,16 @@ done echo echo "Tooling:" -for bin in mise pnpm docker node; do +for bin in mise pnpm docker node git; do if command -v "$bin" >/dev/null; then printf ' %-7s %s\n' "$bin" "$(command -v "$bin")"; else printf ' %-7s missing\n' "$bin"; fi done +# The docs-submodule seed needs partial clone + cone sparse-checkout +# (git >= 2.26); the OS owns git (not mise), so floor-check it here. +_gitv=$(git --version 2>/dev/null | sed 's/[^0-9]*\([0-9]*\.[0-9]*\).*/\1/') +case "$_gitv" in + 1.*|2.[0-9]|2.1[0-9]|2.2[0-5]) echo " git WARNING: $_gitv < 2.26 — too old for the sparse docs-submodule seed (clone-docs)" ;; +esac +unset _gitv if command -v docker >/dev/null; then if docker info >/dev/null 2>&1; then echo " docker (running)"; else echo " docker (not running)"; fi fi diff --git a/workspace/scripts/update.sh b/workspace/scripts/update.sh index c4514e57..01c17255 100755 --- a/workspace/scripts/update.sh +++ b/workspace/scripts/update.sh @@ -1,17 +1,20 @@ #!/usr/bin/env bash -# Update the supabase clone: fetch upstream and REBASE local work — the -# [eval-workspace-*] plumbing commits plus any commits of yours — onto the -# new tip. Uncommitted edits survive via --autostash. A rebase conflict means -# upstream drifted under a plumbing commit (regenerate that patch; see -# workspace/patches/README.md) or under your own commits; the rebase is -# aborted so the workspace is left as found. +# Update the docs submodule (submodules/supabase): fetch upstream and REBASE +# local work — the [eval-workspace-*] plumbing commits plus any commits of +# yours — onto the new tip. Uncommitted edits survive via --autostash. A +# rebase conflict means upstream drifted under a plumbing commit (regenerate +# that patch; see workspace/patches/README.md) or under your own commits; the +# rebase is aborted so the submodule is left as found. # -# mcp and skills are pin-driven submodules (no remote in manifest.json) — -# not managed here. Bump the pin with `git submodule update --remote ` -# and record it via the M3 pin flow, then re-run `mise run apply-patches`. +# The submodule is PINNED by the evals gitlink with `ignore = all`, so a +# rebase here is invisible to `git status` in evals. Recording the bump is a +# deliberate act: git add submodules/supabase && git commit +# +# mcp and agent-skills pins are owned by evals' own pin management (renovate/ +# release flow) — never fetched or rebased here. # # Usage: workspace/scripts/update.sh [--check] [repo...] -# --check fetch + report how far behind each clone is; changes nothing +# --check fetch + report how far behind the submodule is; changes nothing # repos default: every patch-carrying repo from manifest.json set -euo pipefail cd "$(dirname "$0")/../.." @@ -25,24 +28,23 @@ REPOS=("$@") FAILED=0 for repo in ${REPOS[@]+"${REPOS[@]}"}; do dir=$(repo_dir "$repo") - remote=$(repo_remote "$repo") - if [ -z "$remote" ]; then - echo "== $repo: pin-driven ($dir) — not fetched/rebased here." - echo " bump: git submodule update --remote $dir (then record the pin — see M3 pin flow)" + if [ "$repo" != supabase ]; then + echo "== $repo: pin-driven ($dir) — owned by evals pin management, not updated here." continue fi - if [ ! -e "$dir/.git" ]; then echo "== $repo: not cloned — skipping"; continue; fi - branch=$(git -C "$dir" rev-parse --abbrev-ref HEAD) + if [ ! -e "$dir/.git" ]; then echo "== supabase: not seeded — skipping (mise run clone-docs)"; continue; fi + + branch=$(basename "$(git -C "$dir" rev-parse --abbrev-ref origin/HEAD 2>/dev/null || echo origin/master)") old_upstream=$(git -C "$dir" rev-parse "origin/$branch" 2>/dev/null || echo "") - git -C "$dir" fetch -q + git -C "$dir" fetch -q origin behind=$(git -C "$dir" rev-list --count "HEAD..origin/$branch" 2>/dev/null || echo '?') if [ "$CHECK" = 1 ]; then - printf '== %-9s %s@%s is %s commit(s) behind origin/%s\n' "$repo:" "$branch" "$(git -C "$dir" rev-parse --short HEAD)" "$behind" "$branch" + printf '== %-9s %s is %s commit(s) behind origin/%s\n' "$repo:" "$(git -C "$dir" rev-parse --short HEAD)" "$behind" "$branch" continue fi - echo "== $repo: $branch, $behind commit(s) behind — updating" + echo "== $repo: $behind commit(s) behind origin/$branch — updating" git -C "$dir" diff --cached --quiet --ita-visible-in-index \ || { echo " $dir index has staged changes — unstage or commit first" >&2; FAILED=1; continue; } @@ -62,16 +64,15 @@ for repo in ${REPOS[@]+"${REPOS[@]}"}; do new_upstream=$(git -C "$dir" rev-parse "origin/$branch") if [ "$old_upstream" != "$new_upstream" ]; then - case "$repo" in - supabase) - if ! git -C "$dir" diff --quiet "$old_upstream" "$new_upstream" -- pnpm-lock.yaml; then - echo " lockfile changed — pnpm install (docs filter)"; pnpm --dir "$dir" install --filter docs... --silent - fi - if [ -n "$(git -C "$dir" diff --name-only "$old_upstream" "$new_upstream" -- apps/docs/content | head -1)" ]; then - echo " note: docs content changed upstream — the next docs-index re-embeds changed pages (costs cents)" - fi ;; - esac + if ! git -C "$dir" diff --quiet "$old_upstream" "$new_upstream" -- pnpm-lock.yaml; then + echo " lockfile changed — pnpm install (docs filter)"; (cd "$dir" && COREPACK_ENABLE_DOWNLOAD_PROMPT=0 corepack pnpm install --filter docs... --silent) + fi + if [ -n "$(git -C "$dir" diff --name-only "$old_upstream" "$new_upstream" -- apps/docs/content | head -1)" ]; then + echo " note: docs content changed upstream — the next docs-index re-embeds changed pages (costs cents)" + fi fi + echo " pin: the evals gitlink still records the old base (ignore = all hides this)." + echo " record the bump deliberately: git add $dir && git commit" done [ "$FAILED" = 0 ] || { echo; echo "update finished with errors (see above)"; exit 1; } From 5df841525e6e8ec1893aea116a659005df66882e Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 13:53:05 +0200 Subject: [PATCH 02/18] fix: stub @sentry/nextjs in the standalone docs content API Pre-existing drift exposed by the fresh seed: the GraphQL route calls Sentry.captureException/flush (upstream since 2025-06), but under plain tsx (outside Next instrumentation) the package's ESM build resolves without those functions - every request crashed. The old workspace only worked on stale node_modules; its docs-api fails today too. A node:module resolve hook (NODE_OPTIONS --import in docs-api.sh) short- circuits '@sentry/nextjs' to a no-op stub - no upstream-file patch to maintain, and a local dev adapter should not ship telemetry anyway. --- workspace/scripts/sentry-stub-register.mjs | 15 +++++++++++++++ workspace/scripts/sentry-stub.mjs | 8 ++++++++ 2 files changed, 23 insertions(+) create mode 100644 workspace/scripts/sentry-stub-register.mjs create mode 100644 workspace/scripts/sentry-stub.mjs diff --git a/workspace/scripts/sentry-stub-register.mjs b/workspace/scripts/sentry-stub-register.mjs new file mode 100644 index 00000000..992a6151 --- /dev/null +++ b/workspace/scripts/sentry-stub-register.mjs @@ -0,0 +1,15 @@ +// Registers a resolve hook that short-circuits '@sentry/nextjs' to the local +// no-op stub. Injected via NODE_OPTIONS from docs-api.sh; chains with tsx's +// own hooks (ours only intercepts the one specifier). +import { registerHooks } from "node:module"; + +const stubUrl = new URL("./sentry-stub.mjs", import.meta.url).href; + +registerHooks({ + resolve(specifier, context, next) { + if (specifier === "@sentry/nextjs") { + return { url: stubUrl, shortCircuit: true }; + } + return next(specifier, context); + }, +}); diff --git a/workspace/scripts/sentry-stub.mjs b/workspace/scripts/sentry-stub.mjs new file mode 100644 index 00000000..8af0b887 --- /dev/null +++ b/workspace/scripts/sentry-stub.mjs @@ -0,0 +1,8 @@ +// No-op @sentry/nextjs stand-in for the standalone docs content API. +// The route handler calls Sentry.captureException/flush; under plain tsx +// (outside Next's Sentry instrumentation) the real package's ESM build +// resolves without those functions and every request crashes. A local dev +// adapter has no business sending telemetry anyway. Wired up by +// sentry-stub-register.mjs (see docs-api.sh). +export const captureException = () => ''; +export const flush = async () => true; From 9706e47999eef10909ba31c0fe3331851dc561ae Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 13:56:53 +0200 Subject: [PATCH 03/18] fix: Sentry stub via module.register() (Node 20.6+), tsx invoked directly registerHooks() needs Node >= 22.15 while mise pins node "22" - an older 22.x install would fail before startup (review catch). The register() loader API covers all supported nodes. Direct node_modules/.bin/tsx invocation replaces 'corepack pnpm exec': NODE_OPTIONS must reach only the server process - pnpm's own node chokes on the loader hook while probing pnpmfiles. corepack stays install-only (clone-docs/update). Re-verified live: docs-api serves searchDocs from the seeded index. --- workspace/scripts/docs-api.sh | 14 +++++++------- workspace/scripts/sentry-stub-loader.mjs | 9 +++++++++ workspace/scripts/sentry-stub-register.mjs | 17 +++++------------ 3 files changed, 21 insertions(+), 19 deletions(-) create mode 100644 workspace/scripts/sentry-stub-loader.mjs diff --git a/workspace/scripts/docs-api.sh b/workspace/scripts/docs-api.sh index e94d3f2c..a3e6fb0c 100755 --- a/workspace/scripts/docs-api.sh +++ b/workspace/scripts/docs-api.sh @@ -10,20 +10,20 @@ source submodules/supabase/apps/docs/.env.development set +a eval "$(supabase status --workdir submodules/supabase -o env)" : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" -# corepack, cwd inside the submodule: the docs monorepo pins its OWN pnpm -# (packageManager, 11.x) which differs from evals' mise-pinned pnpm — corepack -# resolves the nearest packageManager upward from cwd. Entry path is relative -# to apps/docs (4 levels up to the evals root). The Sentry stub hook no-ops -# the route handler's telemetry calls (see sentry-stub.mjs). +# Runs the locally installed tsx binary directly (no `pnpm exec`: NODE_OPTIONS +# must reach only the server process — pnpm's own node chokes on the loader +# hook during pnpmfile probing). Entry path is relative to apps/docs (4 levels +# up to the evals root). The Sentry stub hook no-ops the route handler's +# telemetry calls (see sentry-stub.mjs). STUB_REGISTER=$PWD/workspace/scripts/sentry-stub-register.mjs cd submodules/supabase/apps/docs +[ -x node_modules/.bin/tsx ] || { echo "tsx not installed — run: mise run clone-docs" >&2; exit 1; } NODE_ENV=development \ NEXT_PUBLIC_SUPABASE_URL="$API_URL" \ NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" \ OPENAI_API_KEY="$OPENAI_API_KEY" \ -COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ NODE_OPTIONS="--import $STUB_REGISTER${NODE_OPTIONS:+ $NODE_OPTIONS}" \ -exec corepack pnpm exec tsx \ +exec node_modules/.bin/tsx \ --conditions=react-server \ --tsconfig tsconfig.json \ ../../../../workspace/scripts/docs-content-api.ts diff --git a/workspace/scripts/sentry-stub-loader.mjs b/workspace/scripts/sentry-stub-loader.mjs new file mode 100644 index 00000000..060c85fa --- /dev/null +++ b/workspace/scripts/sentry-stub-loader.mjs @@ -0,0 +1,9 @@ +// Loader-thread resolve hook: '@sentry/nextjs' -> the no-op stub. +const stubUrl = new URL("./sentry-stub.mjs", import.meta.url).href; + +export async function resolve(specifier, context, next) { + if (specifier === "@sentry/nextjs") { + return { url: stubUrl, shortCircuit: true }; + } + return next(specifier, context); +} diff --git a/workspace/scripts/sentry-stub-register.mjs b/workspace/scripts/sentry-stub-register.mjs index 992a6151..6bd75826 100644 --- a/workspace/scripts/sentry-stub-register.mjs +++ b/workspace/scripts/sentry-stub-register.mjs @@ -1,15 +1,8 @@ // Registers a resolve hook that short-circuits '@sentry/nextjs' to the local // no-op stub. Injected via NODE_OPTIONS from docs-api.sh; chains with tsx's -// own hooks (ours only intercepts the one specifier). -import { registerHooks } from "node:module"; +// own hooks (ours only intercepts the one specifier). Uses module.register() +// (Node 20.6+) rather than registerHooks() (22.15+) — mise pins node "22", +// which an older 22.x install satisfies. +import { register } from "node:module"; -const stubUrl = new URL("./sentry-stub.mjs", import.meta.url).href; - -registerHooks({ - resolve(specifier, context, next) { - if (specifier === "@sentry/nextjs") { - return { url: stubUrl, shortCircuit: true }; - } - return next(specifier, context); - }, -}); +register("./sentry-stub-loader.mjs", import.meta.url); From fb1ad8d97bd8fb2c00f17f645655113401b7614e Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 14:26:59 +0200 Subject: [PATCH 04/18] fix: pin the docs gitlink at the upstream base, not the local marker tip Caught by the cold-install demo: the committed gitlink recorded db16c89, the marker-commit tip of the local submodule (an explicit 'git add submodules/supabase' bypasses ignore = all). Markers never leave the machine (pre-push guard), so every cold seed died with 'upload-pack: not our ref' - and the aborted seed left a half-state (clone done, checkout not) that re-runs never repaired. - gitlink re-pinned at 846706b (the upstream base). - clone-docs.sh: the pin checkout now runs whenever the worktree isn't materialized (fresh seed AND recovery from an aborted one); an unfetchable pin fails loud with the marker-gitlink diagnosis instead of half-seeding. - hooks.test: new check that the gitlink's subject is not an [eval-workspace-*] marker (the accident class ignore = all can't see). --- submodules/supabase | 2 +- workspace/scripts/clone-docs.sh | 25 +++++++++++++++++-------- workspace/scripts/hooks.test.sh | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/submodules/supabase b/submodules/supabase index db16c899..846706bc 160000 --- a/submodules/supabase +++ b/submodules/supabase @@ -1 +1 @@ -Subproject commit db16c89986979822e41bc3b0f0292dc7f52d90eb +Subproject commit 846706bc504d84c0bdae51e6e61d50ccdda86f73 diff --git a/workspace/scripts/clone-docs.sh b/workspace/scripts/clone-docs.sh index ce13bb53..8ab6faae 100755 --- a/workspace/scripts/clone-docs.sh +++ b/workspace/scripts/clone-docs.sh @@ -17,14 +17,12 @@ cd "$(dirname "$0")/../.." SUB=submodules/supabase URL=${SUPABASE_REMOTE:-$(git config -f .gitmodules "submodule.$SUB.url")} -FRESH=0 -if [ -e "$SUB/.git" ]; then - echo "$SUB already seeded; refreshing sparse checkout" -else - FRESH=1 +if [ ! -e "$SUB/.git" ]; then # the gitlink placeholder is an empty dir on fresh checkouts; clone into it git clone --filter=blob:none --no-checkout "$URL" "$SUB" git -C "$SUB" sparse-checkout init --cone +else + echo "$SUB already cloned; refreshing sparse checkout" fi git -C "$SUB" sparse-checkout set \ @@ -36,10 +34,21 @@ git -C "$SUB" sparse-checkout set \ packages/tsconfig packages/ui packages/ui-patterns \ patches supabase -if [ "$FRESH" = 1 ]; then - # check out exactly the pinned SHA (blob filter fetches content on demand) +# Materialize the pin whenever the worktree isn't checked out yet — covers +# fresh seeds AND recovery from an aborted one (clone done, checkout not). +# Never moves HEAD on a checked-out tree: marker commits and your work stay +# put (pin bumps go through `mise run update`). +if [ ! -e "$SUB/apps/docs" ]; then WANT=$(git rev-parse "HEAD:$SUB") - git -C "$SUB" cat-file -e "$WANT^{commit}" 2>/dev/null || git -C "$SUB" fetch -q origin "$WANT" + if ! git -C "$SUB" cat-file -e "$WANT^{commit}" 2>/dev/null; then + git -C "$SUB" fetch -q origin "$WANT" || { + echo "ERROR: the pinned docs SHA $WANT is not fetchable from upstream." >&2 + echo " If its subject is an [eval-workspace-*] marker, the evals gitlink was" >&2 + echo " committed at a local plumbing commit instead of the upstream base —" >&2 + echo " fix the pin (git update-index --cacheinfo 160000,,$SUB)." >&2 + exit 1 + } + fi git -C "$SUB" checkout -q "$WANT" echo "seeded $SUB at $(git -C "$SUB" rev-parse --short HEAD) (the evals pin)" fi diff --git a/workspace/scripts/hooks.test.sh b/workspace/scripts/hooks.test.sh index 4d6876ca..e029483d 100755 --- a/workspace/scripts/hooks.test.sh +++ b/workspace/scripts/hooks.test.sh @@ -55,6 +55,24 @@ for r in $PATCH_REPOS; do test_guard_in "$r" done +# The docs gitlink must record an UPSTREAM sha, never a local [eval-workspace-*] +# plumbing commit: markers can't be pushed (guard above), so a marker gitlink +# makes every cold seed unfetchable ("upload-pack: not our ref"). `ignore = all` +# hides gitlink changes from status but not from an explicit `git add` — this +# is the check that catches that accident. Skipped when the submodule isn't +# seeded (the sha can't be inspected without a clone). +DOCS_SUB=submodules/supabase +if [ -e "$DOCS_SUB/.git" ]; then + _pin=$(git rev-parse "HEAD:$DOCS_SUB" 2>/dev/null || echo "") + if [ -n "$_pin" ] && git -C "$DOCS_SUB" cat-file -e "$_pin^{commit}" 2>/dev/null; then + _subj=$(git -C "$DOCS_SUB" log -1 --format=%s "$_pin") + case "$_subj" in + "[eval-workspace-"*) ck "docs gitlink: pinned at upstream (not a marker)" "marker: $_subj" "upstream" ;; + *) ck "docs gitlink: pinned at upstream (not a marker)" "upstream" "upstream" ;; + esac + fi +fi + echo "hooks.test: $pass passed, $fail failed" if [ "$pass" -eq 0 ] && [ "$fail" -eq 0 ]; then echo "hooks.test: 0 checks ran (no patched repo present — run: mise run setup)" >&2 From a18151b1f95688c267b9ffbb8bab46a387c20c6c Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 14:36:15 +0200 Subject: [PATCH 05/18] fix: ab-demo guide revert used a stale strip prefix (caught by the live demo) The M3 path swap left ${GUIDE#supabase/} unstripped (GUIDE now starts with submodules/), so cleanup passed a nonexistent pathspec to the submodule checkout, '|| true' swallowed it, the canary stayed in the guide, and the 'de-embed' re-indexed the canary. The preflight clean-file check was silently vacuous for the same reason. Both prefixes fixed; a failed revert now warns loudly instead of masking itself. --- workspace/scripts/ab-demo.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workspace/scripts/ab-demo.sh b/workspace/scripts/ab-demo.sh index 1c9a7dcd..8562ef71 100755 --- a/workspace/scripts/ab-demo.sh +++ b/workspace/scripts/ab-demo.sh @@ -36,7 +36,7 @@ docker exec supabase_db_eval-workspace-content true 2>/dev/null || fail "content pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) [ "${pages:-0}" -gt 0 ] 2>/dev/null || fail "docs index not seeded" curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql || fail "docs-api not serving on :3001" -git -C submodules/supabase diff --quiet -- "${GUIDE#supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}" +git -C submodules/supabase diff --quiet -- "${GUIDE#submodules/supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}" [ ! -e "$EVAL_DST" ] || fail "$EVAL_DST already exists — remove it first" echo "This runs a live A/B: 2 claude-sonnet-5 runs + a few embedding cents, ~3-5 min." @@ -46,7 +46,8 @@ read -r -p "Type 'demo' to proceed: " confirmation # --- plant the canary (append-only; reverted by the cleanup trap) --- cleanup() { echo "== demo cleanup: reverting canary, de-embedding, removing throwaway eval ==" - git -C submodules/supabase checkout -- "${GUIDE#supabase/}" 2>/dev/null || true + git -C submodules/supabase checkout -- "${GUIDE#submodules/supabase/}" \ + || echo "WARNING: canary revert FAILED — the de-embed below re-indexes the canary; check ${GUIDE}" >&2 rm -rf "$EVAL_DST" workspace/scripts/docs-index.sh >/dev/null 2>&1 \ || echo "WARNING: cleanup re-embed failed — run \`mise run docs-index\` to de-embed the canary" >&2 From 4c38d6a7f665e94ce835b849770155278df0dce0 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 15:06:42 +0200 Subject: [PATCH 06/18] fix: close the skip->purge hole in the docs embed pipeline Two rows lost their protection when a source went missing: - lint-warnings-skip (LOCAL-ONLY): a skipped source returned [] and vanished before preparation, so purgeOldPages deleted its still-valid rows as stale. The skip now registers its path scope ('/guides/database/database-advisors?%' - the '?' keeps the base guide purgeable) and the purge excludes it. Scoped, not global: a blanket purge suppression would leave treatment-added pages contaminating A/B baselines forever. - index-fail-closed (upstream candidate): fetchPartners() swallowed the query error, so a transient API failure looked like zero partners and the purge deleted every /partners/integrations/* row. It now throws, aborting the run before the purge. Marker commits rebuilt; tree identity + hooks.test (17/0) green. --- workspace/patches/README.md | 4 +- .../supabase-docs-index-fail-closed.patch | 24 +++++++ .../supabase-docs-lint-warnings-skip.patch | 69 ++++++++++++++++++- 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/workspace/patches/README.md b/workspace/patches/README.md index b43680d3..d378b589 100644 --- a/workspace/patches/README.md +++ b/workspace/patches/README.md @@ -28,9 +28,9 @@ derive from the kind). |---|---|---|---|---|---| | `mcp-content-api-url` | `submodules/mcp` | `transports/stdio.ts` | upstream | 3 stdio integration tests (upstream) | `--content-api-url` flag + `SUPABASE_CONTENT_API_URL` — **merged upstream ([mcp#343](https://github.com/supabase/mcp/pull/343), 2026-07-23), not yet in a release**; the patch retires when a release ships the flag and the pin moves past it | | `supabase-content-local-ports` | supabase/supabase | `config.toml` | **LOCAL-ONLY** | n/a | dev ports 55321+ (avoid evals' 54321-9) | -| `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report | +| `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts`, `partner-integrations.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report; partner fetch errors fail loud (a swallowed error looked like zero partners and purged their rows) | | `supabase-docs-guide-checksum` | supabase/supabase | `guideModelLoader.ts` | upstream | none | guide checksum + `tryCatch` `onError` fix (docs-index edit detection) | -| `supabase-docs-lint-warnings-skip` | supabase/supabase | `lint-warnings-guide.ts` | **LOCAL-ONLY** | n/a | `DOCS_EMBED_ALLOW_MISSING_SOURCES` skip | +| `supabase-docs-lint-warnings-skip` | supabase/supabase | `lint-warnings-guide.ts`, `generate-embeddings.ts` | **LOCAL-ONLY** | n/a | `DOCS_EMBED_ALLOW_MISSING_SOURCES` skip; the skipped source registers its path scope and the purge excludes it (skip→purge hole) | | `supabase-docs-reference-dup-sources` | supabase/supabase | `sources/reference-doc.ts` + test | upstream | **unit + real-output tests** | dedupe duplicate reference source paths (the 18-page `inserted 2/1` fix) | ## Changing or regenerating a patch diff --git a/workspace/patches/supabase-docs-index-fail-closed.patch b/workspace/patches/supabase-docs-index-fail-closed.patch index a1322c0e..143c115a 100644 --- a/workspace/patches/supabase-docs-index-fail-closed.patch +++ b/workspace/patches/supabase-docs-index-fail-closed.patch @@ -94,3 +94,27 @@ index d7519d2b15..4dfe6e6ba6 100644 await purgeOldPages(supabaseClient, pageTable, refreshVersion) console.log('Embedding generation complete') +diff --git a/apps/docs/scripts/search/sources/partner-integrations.ts b/apps/docs/scripts/search/sources/partner-integrations.ts +index ad9964722e..b268a3c51c 100644 +--- a/apps/docs/scripts/search/sources/partner-integrations.ts ++++ b/apps/docs/scripts/search/sources/partner-integrations.ts +@@ -21,12 +21,18 @@ function getSupabaseClient() { + + export async function fetchPartners() { + const supabase = getSupabaseClient() +- const { data: partners } = await supabase ++ const { data: partners, error } = await supabase + .from('partners') + .select('slug,overview') + .eq('approved', true) + // We want to show technology integrations, not agencies, in search + .neq('type', 'expert') ++ // Fail loud: swallowing the error makes a transient API failure look like ++ // "zero partners", and the post-index purge then deletes every existing ++ // /partners/integrations/* row as stale. ++ if (error) { ++ throw new Error(`Failed to fetch partners for indexing: ${error.message}`) ++ } + return partners ?? [] + } + diff --git a/workspace/patches/supabase-docs-lint-warnings-skip.patch b/workspace/patches/supabase-docs-lint-warnings-skip.patch index 70a3bf71..941d8b17 100644 --- a/workspace/patches/supabase-docs-lint-warnings-skip.patch +++ b/workspace/patches/supabase-docs-lint-warnings-skip.patch @@ -1,13 +1,78 @@ +diff --git a/apps/docs/scripts/search/generate-embeddings.ts b/apps/docs/scripts/search/generate-embeddings.ts +index 4dfe6e6ba6..cb9166c1f2 100644 +--- a/apps/docs/scripts/search/generate-embeddings.ts ++++ b/apps/docs/scripts/search/generate-embeddings.ts +@@ -18,6 +18,7 @@ import { + logFailedSections, + } from './embeddings/utils.js' + import { fetchAllSources } from './sources/index.js' ++import { skippedSourcePathScopes } from './sources/lint-warnings-guide.js' + + const CONFIG = { + // OpenAI settings +@@ -532,7 +533,7 @@ async function generateEmbeddings() { + ) + } + +- await purgeOldPages(supabaseClient, pageTable, refreshVersion) ++ await purgeOldPages(supabaseClient, pageTable, refreshVersion, skippedSourcePathScopes) + + console.log('Embedding generation complete') + } +@@ -591,13 +592,22 @@ function logFailedPages(pageInfoMap: Map, processingResult: Pr + async function purgeOldPages( + supabaseClient: SupabaseClient, + pageTable: string, +- refreshVersion: string ++ refreshVersion: string, ++ protectPathScopes: string[] = [] + ) { + console.log(`Removing old pages and their sections`) +- const { error: deletePageError } = await supabaseClient +- .from(pageTable) +- .delete() +- .filter('version', 'neq', refreshVersion) ++ // A source skipped under DOCS_EMBED_ALLOW_MISSING_SOURCES never entered ++ // preparation, so its still-valid rows carry an old version — exclude its ++ // path scope from the purge instead of deleting them. Everything else ++ // (removed/renamed pages) purges as usual, keeping A/B baselines clean. ++ if (protectPathScopes.length > 0) { ++ console.warn(`Purge excluding skipped source path scope(s): ${protectPathScopes.join(', ')}`) ++ } ++ let query = supabaseClient.from(pageTable).delete().filter('version', 'neq', refreshVersion) ++ for (const scope of protectPathScopes) { ++ query = query.not('path', 'like', scope) ++ } ++ const { error: deletePageError } = await query + if (deletePageError) throw deletePageError + } + diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts -index d28a57a27f..0207f87504 100644 +index d28a57a27f..03986df4ca 100644 --- a/apps/docs/scripts/search/sources/lint-warnings-guide.ts +++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts -@@ -29,6 +29,10 @@ export class LintWarningsGuideLoader extends BaseLoader { +@@ -13,6 +13,12 @@ const privateKey = process.env.DOCS_GITHUB_APP_PRIVATE_KEY + + const getBasename = (path: string) => path.split('/').at(-1)!.replace(/\.md$/, '') + ++// Path scopes (SQL LIKE patterns) of sources skipped under ++// DOCS_EMBED_ALLOW_MISSING_SOURCES. generate-embeddings excludes these from ++// the old-pages purge: a skipped source never enters preparation, so its ++// still-valid rows carry an old version and would otherwise be deleted. ++export const skippedSourcePathScopes: string[] = [] ++ + export class LintWarningsGuideLoader extends BaseLoader { + type = 'markdown' as const + +@@ -29,6 +35,13 @@ export class LintWarningsGuideLoader extends BaseLoader { async load() { if (!appId || !installationId || !privateKey) { + if (process.env.DOCS_EMBED_ALLOW_MISSING_SOURCES) { + console.warn('Skipping lint-warnings guide source: DOCS_GITHUB_APP_* not set (DOCS_EMBED_ALLOW_MISSING_SOURCES)') ++ // Protect this loader's pages (`?queryGroups=lint&lint=`) ++ // from the purge; the `?` keeps the base guide page purgeable. ++ skippedSourcePathScopes.push(`${this.path}?%`) + return [] + } throw new Error('Missing DOCS_GITHUB_APP_* environment variables') From 2132df6fb83e2dffe86030da1ab044b9e11d26c3 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 15:12:01 +0200 Subject: [PATCH 07/18] fix: embed-env patch check by marker commit, not textual reverse-apply The reverse-apply check broke as soon as a later patch legitimately touched the same lines (lint-warnings-skip now extends fail-closed's purgeOldPages), failing every docs-seed/index on a correctly-patched tree. Check the marker-commit subject instead - the same invariant apply-patches verifies with tree identity. Caught by cold run #2. --- workspace/scripts/docs-embed-env.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/workspace/scripts/docs-embed-env.sh b/workspace/scripts/docs-embed-env.sh index f4071bf9..405a05e3 100644 --- a/workspace/scripts/docs-embed-env.sh +++ b/workspace/scripts/docs-embed-env.sh @@ -2,7 +2,12 @@ # fail-closed patch check, key loading, docs env, local-stack credentials, and # the OpenAI preflight. Source from the repo root (the callers cd there first); # not executable on its own. -if ! git -C submodules/supabase apply --reverse --check "$PWD/workspace/patches/supabase-docs-index-fail-closed.patch" 2>/dev/null; then +# Marker-commit check (same invariant apply-patches verifies with tree +# identity). NOT a textual reverse-apply: stacked patches legitimately touch +# the same lines (lint-warnings-skip extends fail-closed's purge), which +# breaks reverse-apply against the final tree while the plumbing is fine. +if ! git -C submodules/supabase log --format=%s HEAD --not --remotes 2>/dev/null \ + | grep -qxF '[eval-workspace-upstream] supabase-docs-index-fail-closed'; then echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 exit 1 fi From 23d9ac766ee6bd34d106e8c75a0fb3145fe3ca10 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 15:13:36 +0200 Subject: [PATCH 08/18] fix: marker check must consume the log stream (pipefail SIGPIPE hazard) --- workspace/scripts/docs-embed-env.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workspace/scripts/docs-embed-env.sh b/workspace/scripts/docs-embed-env.sh index 405a05e3..70c29ea3 100644 --- a/workspace/scripts/docs-embed-env.sh +++ b/workspace/scripts/docs-embed-env.sh @@ -6,8 +6,10 @@ # identity). NOT a textual reverse-apply: stacked patches legitimately touch # the same lines (lint-warnings-skip extends fail-closed's purge), which # breaks reverse-apply against the final tree while the plumbing is fine. +# grep consumes the whole stream (no -q): early exit would SIGPIPE git log +# under the callers' pipefail and falsely fail the check. if ! git -C submodules/supabase log --format=%s HEAD --not --remotes 2>/dev/null \ - | grep -qxF '[eval-workspace-upstream] supabase-docs-index-fail-closed'; then + | grep -xF '[eval-workspace-upstream] supabase-docs-index-fail-closed' >/dev/null; then echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 exit 1 fi From ce467ace8b1c41ade345fecb412639c9a493a995 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 15:19:22 +0200 Subject: [PATCH 09/18] fix: partners source skips with purge protection under DOCS_EMBED_ALLOW_MISSING_SOURCES The fail-loud partners fix immediately caught a real, previously silent failure: the misc API rejects the legacy anon key locally ('Legacy API keys are disabled'), so partners has ALWAYS quietly indexed as zero on this machine - strong candidate for the 27 purged rows. Upstream keeps the hard throw (abort before purge); locally the ALLOW flag now treats partners like lint-warnings: warn, register '/partners/integrations/%' in the shared scope registry (moved to base.ts, both sources push), and skip. Caught by cold run #3. --- workspace/patches/README.md | 2 +- .../supabase-docs-lint-warnings-skip.patch | 63 +++++++++++++++---- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/workspace/patches/README.md b/workspace/patches/README.md index d378b589..a98a5043 100644 --- a/workspace/patches/README.md +++ b/workspace/patches/README.md @@ -30,7 +30,7 @@ derive from the kind). | `supabase-content-local-ports` | supabase/supabase | `config.toml` | **LOCAL-ONLY** | n/a | dev ports 55321+ (avoid evals' 54321-9) | | `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts`, `partner-integrations.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report; partner fetch errors fail loud (a swallowed error looked like zero partners and purged their rows) | | `supabase-docs-guide-checksum` | supabase/supabase | `guideModelLoader.ts` | upstream | none | guide checksum + `tryCatch` `onError` fix (docs-index edit detection) | -| `supabase-docs-lint-warnings-skip` | supabase/supabase | `lint-warnings-guide.ts`, `generate-embeddings.ts` | **LOCAL-ONLY** | n/a | `DOCS_EMBED_ALLOW_MISSING_SOURCES` skip; the skipped source registers its path scope and the purge excludes it (skip→purge hole) | +| `supabase-docs-lint-warnings-skip` | supabase/supabase | `base.ts`, `lint-warnings-guide.ts`, `partner-integrations.ts`, `generate-embeddings.ts` | **LOCAL-ONLY** | n/a | `DOCS_EMBED_ALLOW_MISSING_SOURCES` skip; the skipped source registers its path scope and the purge excludes it (skip→purge hole) | | `supabase-docs-reference-dup-sources` | supabase/supabase | `sources/reference-doc.ts` + test | upstream | **unit + real-output tests** | dedupe duplicate reference source paths (the 18-page `inserted 2/1` fix) | ## Changing or regenerating a patch diff --git a/workspace/patches/supabase-docs-lint-warnings-skip.patch b/workspace/patches/supabase-docs-lint-warnings-skip.patch index 941d8b17..32777377 100644 --- a/workspace/patches/supabase-docs-lint-warnings-skip.patch +++ b/workspace/patches/supabase-docs-lint-warnings-skip.patch @@ -1,12 +1,12 @@ diff --git a/apps/docs/scripts/search/generate-embeddings.ts b/apps/docs/scripts/search/generate-embeddings.ts -index 4dfe6e6ba6..cb9166c1f2 100644 +index 4dfe6e6ba6..cf6d80b683 100644 --- a/apps/docs/scripts/search/generate-embeddings.ts +++ b/apps/docs/scripts/search/generate-embeddings.ts @@ -18,6 +18,7 @@ import { logFailedSections, } from './embeddings/utils.js' import { fetchAllSources } from './sources/index.js' -+import { skippedSourcePathScopes } from './sources/lint-warnings-guide.js' ++import { skippedSourcePathScopes } from './sources/base.js' const CONFIG = { // OpenAI settings @@ -47,24 +47,34 @@ index 4dfe6e6ba6..cb9166c1f2 100644 if (deletePageError) throw deletePageError } -diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts -index d28a57a27f..03986df4ca 100644 ---- a/apps/docs/scripts/search/sources/lint-warnings-guide.ts -+++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts -@@ -13,6 +13,12 @@ const privateKey = process.env.DOCS_GITHUB_APP_PRIVATE_KEY - - const getBasename = (path: string) => path.split('/').at(-1)!.replace(/\.md$/, '') +diff --git a/apps/docs/scripts/search/sources/base.ts b/apps/docs/scripts/search/sources/base.ts +index a7835ccd92..c0bb41ead7 100644 +--- a/apps/docs/scripts/search/sources/base.ts ++++ b/apps/docs/scripts/search/sources/base.ts +@@ -31,3 +31,9 @@ export abstract class BaseSource { + abstract extractIndexedContent(): string + } ++ +// Path scopes (SQL LIKE patterns) of sources skipped under +// DOCS_EMBED_ALLOW_MISSING_SOURCES. generate-embeddings excludes these from +// the old-pages purge: a skipped source never enters preparation, so its +// still-valid rows carry an old version and would otherwise be deleted. +export const skippedSourcePathScopes: string[] = [] -+ - export class LintWarningsGuideLoader extends BaseLoader { - type = 'markdown' as const +diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts +index d28a57a27f..0a16468494 100644 +--- a/apps/docs/scripts/search/sources/lint-warnings-guide.ts ++++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts +@@ -3,7 +3,7 @@ import { Octokit } from '@octokit/core' + import { retry } from '@octokit/plugin-retry' + import crypto, { createHash } from 'node:crypto' + import { OCTOKIT_RETRY_OPTIONS } from '../../../lib/octokit.constants.js' +-import { BaseLoader, BaseSource } from './base.js' ++import { BaseLoader, BaseSource, skippedSourcePathScopes } from './base.js' -@@ -29,6 +35,13 @@ export class LintWarningsGuideLoader extends BaseLoader { + const RetryOctokit = Octokit.plugin(retry) + +@@ -29,6 +29,13 @@ export class LintWarningsGuideLoader extends BaseLoader { async load() { if (!appId || !installationId || !privateKey) { @@ -78,3 +88,30 @@ index d28a57a27f..03986df4ca 100644 throw new Error('Missing DOCS_GITHUB_APP_* environment variables') } +diff --git a/apps/docs/scripts/search/sources/partner-integrations.ts b/apps/docs/scripts/search/sources/partner-integrations.ts +index b268a3c51c..2f9e970cb1 100644 +--- a/apps/docs/scripts/search/sources/partner-integrations.ts ++++ b/apps/docs/scripts/search/sources/partner-integrations.ts +@@ -1,7 +1,7 @@ + import { type SupabaseClient, createClient } from '@supabase/supabase-js' + import { upperFirst } from 'lodash-es' + import { processMdx } from '../../helpers.mdx.js' +-import { BaseLoader, BaseSource } from './base.js' ++import { BaseLoader, BaseSource, skippedSourcePathScopes } from './base.js' + + type PartnerData = { + slug: string // The partner slug corresponding to the last part of the URL +@@ -31,6 +31,13 @@ export async function fetchPartners() { + // "zero partners", and the post-index purge then deletes every existing + // /partners/integrations/* row as stale. + if (error) { ++ if (process.env.DOCS_EMBED_ALLOW_MISSING_SOURCES) { ++ console.warn( ++ `Skipping partner integrations source: ${error.message} (DOCS_EMBED_ALLOW_MISSING_SOURCES)` ++ ) ++ skippedSourcePathScopes.push('/partners/integrations/%') ++ return [] ++ } + throw new Error(`Failed to fetch partners for indexing: ${error.message}`) + } + return partners ?? [] From 8c57a8012176fb103edfdd64efa57a771e497dbc Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 15:50:45 +0200 Subject: [PATCH 10/18] review: receipt coverage, vacuous test, dead clone concept (thermo pass) Self-review findings on this branch, all verified live: - provenance.mjs recorded working-tree state only for non-submodule entries - now the empty set, so the docs submodule's dirty state (the very thing a docs A/B varies) silently left receipts, and mcp never had it. The model was wrong: what earns a tree record is 'carries patches' (a deliberately floating tree), not 'not a submodule'. submodules. now enumerates from git (status --cached) and records the PIN, so a new submodule can't drop out of receipts; repos. records working reality for patch repos. hooks.test asserts both (19/0). - ab.test's docs mixed-path check still used the pre-M3 path, so it exited 2 at loop selection instead of the scope check it claims to test - vacuous. Fixed + a positive docs-mapping assertion (24/0). - docs-embed-env hardcoded the marker subject; patches-lib owns that format (patch_subject) - now reused. - The 'plain clone' concept is dead: repo_remote() had zero callers, manifest.mjs carried a dead schema branch + stale comments. Deleted; kind: submodule is now required. --- workspace/scripts/ab.test.sh | 8 +++++-- workspace/scripts/docs-embed-env.sh | 6 ++++-- workspace/scripts/hooks.test.sh | 12 ++++++++++- workspace/scripts/manifest.mjs | 29 ++++++++++++------------- workspace/scripts/patches-lib.sh | 11 +++++----- workspace/scripts/provenance.mjs | 33 ++++++++++++++++------------- 6 files changed, 57 insertions(+), 42 deletions(-) diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh index c1ad810f..7005a18f 100755 --- a/workspace/scripts/ab.test.sh +++ b/workspace/scripts/ab.test.sh @@ -83,8 +83,12 @@ ck "edit intact after restore-sync failure" "$(git -C "$SK" hash-object "$REL")" ck "reports re-sync error" "$(grep -c 'post-restore re-sync failed' /tmp/ab_selftest3.out)" "1" ck "no A/B report on failed restore" "$(grep -c '=== A/B' /tmp/ab_selftest3.out)" "0" -# --- docs loop must reject a second path outside the content scope --- -rc4=$(AB_DRYRUN=1 bash workspace/scripts/ab.sh e x supabase/apps/docs/content/a.mdx supabase/apps/studio/foo.ts >/dev/null 2>&1; echo $?) +# --- docs loop: path mapping resolves, and a second path outside the content +# scope is rejected AT THE SCOPE CHECK (paths must be current — a stale prefix +# here once turned this into a vacuous always-exit-2 test) --- +out4=$(AB_DRYRUN=1 bash workspace/scripts/ab.sh e x submodules/supabase/apps/docs/content/a.mdx 2>&1 || true) +ck "docs path maps to the docs loop" "$(printf '%s' "$out4" | grep -c 'loop=docs clone=submodules/supabase')" "1" +rc4=$(AB_DRYRUN=1 bash workspace/scripts/ab.sh e x submodules/supabase/apps/docs/content/a.mdx submodules/supabase/apps/studio/foo.ts >/dev/null 2>&1; echo $?) ck "rejects non-content path in docs loop" "$rc4" "2" # --- dirty clone index (e.g. intent-to-add residue) must be refused before stashing --- diff --git a/workspace/scripts/docs-embed-env.sh b/workspace/scripts/docs-embed-env.sh index 70c29ea3..e347aa2a 100644 --- a/workspace/scripts/docs-embed-env.sh +++ b/workspace/scripts/docs-embed-env.sh @@ -6,10 +6,12 @@ # identity). NOT a textual reverse-apply: stacked patches legitimately touch # the same lines (lint-warnings-skip extends fail-closed's purge), which # breaks reverse-apply against the final tree while the plumbing is fine. +# Subject string comes from patches-lib (the one owner of the marker format). # grep consumes the whole stream (no -q): early exit would SIGPIPE git log # under the callers' pipefail and falsely fail the check. -if ! git -C submodules/supabase log --format=%s HEAD --not --remotes 2>/dev/null \ - | grep -xF '[eval-workspace-upstream] supabase-docs-index-fail-closed' >/dev/null; then +source workspace/scripts/patches-lib.sh +if ! git -C "$(repo_dir supabase)" log --format=%s HEAD --not --remotes 2>/dev/null \ + | grep -xF "$(patch_subject supabase-docs-index-fail-closed)" >/dev/null; then echo 'ERROR: fail-closed index patch is not applied; run workspace/scripts/apply-patches.sh' >&2 exit 1 fi diff --git a/workspace/scripts/hooks.test.sh b/workspace/scripts/hooks.test.sh index e029483d..4ec4866b 100755 --- a/workspace/scripts/hooks.test.sh +++ b/workspace/scripts/hooks.test.sh @@ -61,7 +61,7 @@ done # hides gitlink changes from status but not from an explicit `git add` — this # is the check that catches that accident. Skipped when the submodule isn't # seeded (the sha can't be inspected without a clone). -DOCS_SUB=submodules/supabase +DOCS_SUB=$(repo_dir supabase) if [ -e "$DOCS_SUB/.git" ]; then _pin=$(git rev-parse "HEAD:$DOCS_SUB" 2>/dev/null || echo "") if [ -n "$_pin" ] && git -C "$DOCS_SUB" cat-file -e "$_pin^{commit}" 2>/dev/null; then @@ -73,6 +73,16 @@ if [ -e "$DOCS_SUB/.git" ]; then fi fi +# Receipt coverage: every configured submodule's pin must appear (derived +# from git, so a new submodule can't silently drop out), and patch-carrying +# repos must carry working-tree records — per-arm A/B receipts differ by +# exactly the edit under test only if the working tree is recorded. +_receipt=$(node workspace/scripts/provenance.mjs 2>/dev/null || echo '{}') +ck "receipt: docs submodule pin recorded" \ + "$(printf '%s' "$_receipt" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{const r=JSON.parse(d);console.log(/^[0-9a-f]{40}$/.test(r.submodules?.supabase||"")?"y":"n")})')" "y" +ck "receipt: patch repos carry tree records" \ + "$(printf '%s' "$_receipt" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{const r=JSON.parse(d);console.log(("supabase" in (r.repos||{}))&&("mcp" in (r.repos||{}))?"y":"n")})')" "y" + echo "hooks.test: $pass passed, $fail failed" if [ "$pass" -eq 0 ] && [ "$fail" -eq 0 ]; then echo "hooks.test: 0 checks ran (no patched repo present — run: mise run setup)" >&2 diff --git a/workspace/scripts/manifest.mjs b/workspace/scripts/manifest.mjs index 142965f5..2c0d9e2d 100755 --- a/workspace/scripts/manifest.mjs +++ b/workspace/scripts/manifest.mjs @@ -1,15 +1,14 @@ #!/usr/bin/env node // Loader + CLI for manifest.json (lives at workspace/manifest.json, beside // scripts/) — the single source of truth for the repos this workspace wires -// together: repo-root-relative checkout dirs, upstream remotes (clone -// entries only), and the ordered enabler-patch set per repo (patch names -// without the patches/ prefix or .patch suffix; "localPatches" marks the dev -// shims that must never reach an upstream PR — see patches/README.md). -// `kind: "submodule"` repos (mcp, skills) have no remote — .gitmodules owns -// that — and are never cloned by our scripts; a repo without `kind` is a -// plain clone and MUST have a remote. Every `dir` is repo-root-relative; -// resolving it is the caller's job (bash scripts `cd` to the repo root, -// two levels up from workspace/scripts, before using it). +// together: repo-root-relative checkout dirs and the ordered enabler-patch +// set per repo (patch names without the patches/ prefix or .patch suffix; +// "localPatches" marks the dev shims that must never reach an upstream PR — +// see patches/README.md). Every repo is a pinned submodule (.gitmodules owns +// remotes; nothing here is cloned by our scripts — the docs submodule is +// seeded by clone-docs.sh from the .gitmodules url). Every `dir` is +// repo-root-relative; resolving it is the caller's job (bash scripts `cd` to +// the repo root, two levels up from workspace/scripts, before using it). // // loadManifest() is THE loader: every consumer (the CLI below, the bash // facade scripts/patches-lib.sh through it, and provenance.mjs via import) @@ -44,13 +43,11 @@ export function loadManifest() { } for (const [name, repo] of Object.entries(manifest.repos)) { if (!isStr(repo?.dir)) throw new Error(`repos.${name}.dir must be a non-empty string`); - if (repo.kind !== undefined && repo.kind !== "submodule") { - throw new Error(`repos.${name}.kind must be "submodule" when present`); + if (repo.kind !== "submodule") { + throw new Error(`repos.${name}.kind must be "submodule" (every repo is a pinned submodule; the clone kind is gone)`); } - if (repo.kind === "submodule") { - if (repo.remote !== undefined) throw new Error(`repos.${name}.remote must be absent for kind:"submodule" (.gitmodules owns the remote)`); - } else if (!isStr(repo.remote)) { - throw new Error(`repos.${name}.remote must be a non-empty string (required for clone entries)`); + if (repo.remote !== undefined) { + throw new Error(`repos.${name}.remote must be absent (.gitmodules owns the remote)`); } if (repo.patches !== undefined && !isStrArr(repo.patches)) throw new Error(`repos.${name}.patches must be a non-empty array of strings`); if (repo.localPatches !== undefined) { @@ -106,7 +103,7 @@ if (isMain) { const [name, key] = args; const repo = manifest.repos[name ?? ""]; if (!repo) die(`unknown repo "${name ?? ""}"`); - if (!["dir", "remote", "patches", "localPatches"].includes(key ?? "")) die(`unknown key "${key ?? ""}"`); + if (!["dir", "patches", "localPatches"].includes(key ?? "")) die(`unknown key "${key ?? ""}"`); const v = repo[key]; if (v !== undefined) console.log(Array.isArray(v) ? v.join("\n") : v); break; diff --git a/workspace/scripts/patches-lib.sh b/workspace/scripts/patches-lib.sh index 0e491319..6fef4745 100644 --- a/workspace/scripts/patches-lib.sh +++ b/workspace/scripts/patches-lib.sh @@ -1,9 +1,9 @@ # Bash facade over manifest.json — the single source of truth for repos, -# remotes, and enabler patches (order and kind; marker subjects derive from -# the kind). All data queries go through scripts/manifest.mjs, which validates -# the schema and fails loud. Sourced by apply-patches.sh, update.sh, -# publish.sh, status.sh, setup.sh, clone-docs.sh. Requires bash (BASH_SOURCE) -# and node (pinned via mise). +# checkout dirs, and enabler patches (order and kind; marker subjects derive +# from the kind). All data queries go through scripts/manifest.mjs, which +# validates the schema and fails loud. Sourced by apply-patches.sh, update.sh, +# publish.sh, status.sh, setup.sh, docs-embed-env.sh, hooks.test.sh. Requires +# bash (BASH_SOURCE) and node (pinned via mise). _MANIFEST_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) _manifest() { node "$_MANIFEST_LIB_DIR/manifest.mjs" "$@"; } @@ -29,7 +29,6 @@ done unset _r _MANIFEST_REPOS repo_dir() { local d; d=$(_manifest get "$1" dir); echo "${d:-$1}"; } -repo_remote() { _manifest get "$1" remote; } patches_for() { local out="" n diff --git a/workspace/scripts/provenance.mjs b/workspace/scripts/provenance.mjs index 93f42820..d71632ea 100755 --- a/workspace/scripts/provenance.mjs +++ b/workspace/scripts/provenance.mjs @@ -23,8 +23,8 @@ import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { loadManifest } from "./manifest.mjs"; -// This is the evals repo root (workspace/scripts -> ../..): supabase/, -// submodules/, and .docs-index-stamp.json all live here. The tracked-patches +// This is the evals repo root (workspace/scripts -> ../..): submodules/ +// and .docs-index-stamp.json live here. The tracked-patches // directory is a level lower, under workspace/ (see workspaceRoot below). const root = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const workspaceRoot = join(root, "workspace"); @@ -110,18 +110,19 @@ const repoState = (dir) => { }; }; -// The two pinned submodules, keyed the way ab.sh/status.sh --json expect -// (basename of the submodules/ path). `git submodule status` prefixes an -// uninitialized entry with "-" — its SHA there is only the pinned index -// entry, not a real checkout, so that reads as null, not a fake SHA. +// Every configured submodule, keyed the way ab.sh/status.sh --json expect +// (basename of the submodules/ path). `--cached` reports the gitlink sha +// recorded in the superproject — what this checkout PROMISES — enumerated +// straight from git so a new submodule can never silently drop out of +// receipts. Patch-carrying trees deliberately float above the pin (markers +// + your uncommitted edit); repos. records that working-tree reality. const submoduleShas = () => { - const shas = { "agent-skills": null, mcp: null }; - for (const line of (git(".", "submodule", "status") ?? "").split("\n")) { + const shas = {}; + for (const line of (git(".", "submodule", "status", "--cached") ?? "").split("\n")) { if (!line) continue; - const initialized = line[0] !== "-"; const [sha, subPath] = line.slice(1).trim().split(/\s+/); const name = subPath?.split("/").pop(); - if (name && name in shas) shas[name] = initialized ? sha : null; + if (name) shas[name] = sha; } return shas; }; @@ -139,15 +140,17 @@ const buildProvenance = () => { }; const submodules = submoduleShas(); - // Only non-submodule entries (today: supabase) get a full clone-shaped - // `repos.` record — the two submodules are already covered above by - // exact pinned SHA, which is what a submodule actually promises. + // A pinned-sha record is what a plain submodule promises (skills). But a + // PATCH-CARRYING repo (mcp, supabase) deliberately floats its working tree + // above the pin — marker commits plus your uncommitted A/B edit — so those + // get a full working-tree record too: per-arm receipts must differ by + // exactly the edit under test, and the pin sha alone can't show that. const repos = {}; const patches = {}; for (const [name, spec] of Object.entries(manifest.repos)) { const initialized = existsSync(join(root, spec.dir, ".git")); - if (spec.kind !== "submodule") { - repos[name] = initialized ? { dir: spec.dir, cloned: true, ...repoState(spec.dir) } : { dir: spec.dir, cloned: false }; + if (spec.patches?.length) { + repos[name] = initialized ? { dir: spec.dir, initialized: true, ...repoState(spec.dir) } : { dir: spec.dir, initialized: false }; } if (!spec.patches?.length || !initialized) continue; From 2905cc5ddbcf503ee552b1a0f9f2b34994f55df5 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 17:03:25 +0200 Subject: [PATCH 11/18] fix: A/B cancellation recovery + pre-spend seeded-index gate Two gaps observed live during warm-demo runs: - A hard kill mid-baseline bypassed the EXIT trap and stranded the user's edit in an unmarked stash; the next run then failed with a cryptic 'no unstaged edit'. Now: catchable signals (HUP/INT/TERM) route through exit so the EXIT trap restores exactly once (never attached directly - signal + EXIT would double-pop); the baseline stash carries an identifying message; and preflight detects a stranded marked stash and prints the exact recovery command (covers SIGKILL, which no trap can). Proven live: TERM mid-baseline restores the edit with no stranded stash. 3 new ab.test checks (27/0). - The docs preflight checked docs-api but not the index: an empty-but-running DB (another session's teardown, volume recreated) would turn the treatment sync into a full paid seed instead of failing pre-spend. Now refused with a pointer at docs-seed and the no-args readiness probe. README: restore guarantee stated precisely (incl. the SIGKILL case) + discriminator-eval design tip (vague questions run for minutes). --- workspace/README.md | 13 +++++++++---- workspace/scripts/ab.sh | 24 +++++++++++++++++++++++- workspace/scripts/ab.test.sh | 9 +++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/workspace/README.md b/workspace/README.md index 6406b56f..f5c217f1 100644 --- a/workspace/README.md +++ b/workspace/README.md @@ -58,10 +58,15 @@ loop's scope, then mise run ab # treatment (edit applied) vs baseline (edit reverted) ``` -The edit is always restored (a failed restore fails the run loudly); per-arm -provenance receipts land in `results-ab/*.json`. Cost: two model runs. No -args = readiness probe. First time? `mise run ab-demo` is a guided, -self-cleaning live proof on the docs loop (spend-gated, asks first). +The edit is restored on every exit path, including Ctrl-C/TERM (a failed +restore fails the run loudly). The one uncatchable case is SIGKILL mid-run: +the edit lands in a marked stash, and the next `ab` invocation detects it and +prints the recovery command. Per-arm provenance receipts land in +`results-ab/*.json`. Cost: two model runs. No args = readiness probe. First +time? `mise run ab-demo` is a guided, self-cleaning live proof on the docs +loop (spend-gated, asks first). Writing your own discriminator eval? Ask for +one precise, docs-only fact ("name the exact package for X") — vague +questions make both arms search for minutes before converging. ## Patches & publishing diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh index ed14dc71..ad5a7bf6 100755 --- a/workspace/scripts/ab.sh +++ b/workspace/scripts/ab.sh @@ -75,6 +75,17 @@ if [ -n "${AB_DRYRUN:-}" ]; then fi : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY not in keychain — see README}" +# A previous run killed mid-baseline (SIGKILL beats any trap) leaves the edit +# in a stash marked with AB_STASH_MSG. Detect it BEFORE the no-unstaged-edit +# check below turns that into a cryptic "nothing to A/B". +AB_STASH_MSG="eval-workspace ab baseline stash" +if git -C "$CLONE" stash list | grep -F "$AB_STASH_MSG" >/dev/null; then + echo "a previous A/B was interrupted mid-baseline — your edit is stranded in a stash." >&2 + echo "recover it, then re-run:" >&2 + echo " git -C $CLONE stash pop" >&2 + echo " mise run ab $EVAL ${PATHS[0]} # (docs edits: the pop leaves the index at baseline until the next run re-embeds)" >&2 + exit 1 +fi # A/B reverts tracked, unstaged working-tree edits only, via a scoped git stash. # `stash push -- ` merges whole-index state, so ANY staged entry in the # clone (even an unrelated intent-to-add) breaks it. Require a clean index. @@ -92,6 +103,10 @@ done if [ "$LOOP" = docs ]; then : "${OPENAI_API_KEY:?OPENAI_API_KEY not in keychain — the docs re-embed needs it}" curl -sf -o /dev/null "$CONTENT_URL" || { echo "docs-api not reachable on :3001 — run \`mise run docs-api\` in another terminal first" >&2; exit 1; } + # Refuse BEFORE spending: an empty-but-running DB would otherwise turn the + # treatment sync into a full paid seed instead of an incremental re-embed. + pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) + [ "${pages:-0}" -gt 0 ] 2>/dev/null || { echo "docs index is not seeded (page count: ${pages:-0}) — run \`mise run docs-seed\` once first (mise run ab with no args = full readiness probe)" >&2; exit 1; } [ -d "$MCP/dist" ] || { echo "building local mcp (needed for search_docs routing)…"; ( cd submodules/mcp && pnpm install && pnpm build ); } fi @@ -134,12 +149,19 @@ restore() { } # Preserve the run's own failure status; a clean run that fails cleanup exits nonzero. trap 'rc=$?; if ! restore && [ "$rc" -eq 0 ]; then rc=1; fi; exit $rc' EXIT +# Catchable signals route through `exit`, so the EXIT trap restores exactly +# once (never attach restore to the signals directly: signal + EXIT would +# double-pop). SIGKILL is uncatchable — that case is covered by the marked +# stash + the preflight detection above. +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM echo "== treatment: edit applied (${PATHS[*]}) ==" run_eval treatment echo "== reverting edit for baseline ==" -git -C "$CLONE" stash push -q -- "${REL[@]}" +git -C "$CLONE" stash push -q -m "$AB_STASH_MSG" -- "${REL[@]}" STASHED=1 echo "== baseline: edit reverted ==" diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh index 7005a18f..85686286 100755 --- a/workspace/scripts/ab.test.sh +++ b/workspace/scripts/ab.test.sh @@ -50,6 +50,15 @@ printf '\n\n' >> "$FILE" edited=$(git -C "$SK" hash-object "$REL") index0=$(git -C "$SK" diff --cached -- "$REL") # empty (was clean) +# --- a stranded baseline stash (SIGKILL mid-run) must be detected pre-spend +# with the recovery command, BEFORE the no-unstaged-edit check muddies it --- +git -C "$SK" stash push -q -m "eval-workspace ab baseline stash" -- "$REL" +out0=$(ANTHROPIC_API_KEY=dummy bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" 2>&1; echo "rc=$?") +ck "stranded stash refused" "$(printf '%s' "$out0" | grep -c 'interrupted mid-baseline')" "1" +ck "stranded stash names recovery" "$(printf '%s' "$out0" | grep -c "git -C $SK stash pop")" "1" +ck "stranded stash exits nonzero" "$(printf '%s' "$out0" | grep -c 'rc=1')" "1" +git -C "$SK" stash pop -q # put the edit back for the tests below + # observable sync hook: every sync appends a line (treatment, baseline, restore = 3) CNT=/tmp/ab_selftest_sync.cnt SYNC="echo x >> $CNT" From 5a57d5da62534d3423370a3341de335b15818b77 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 17:10:44 +0200 Subject: [PATCH 12/18] fix: exact stash ref in recovery; zero-cost eval validation preflight Two review catches on the cancellation hardening: - The recovery message printed a plain 'git stash pop', which takes the NEWEST stash - a user's own unrelated stash above the stranded one would get popped instead. Detection now resolves and prints the exact marked ref(s), highest index first so remaining refs stay stable. Tested with a decoy stash above the stranded one (ab.test 28/0). (Also: the refactor moved the detection grep out of an if-condition, where an empty stash list + set -e silently killed every run - guarded.) - Eval schema/typo errors surfaced only at harness discovery, AFTER the treatment re-embed had spent (observed live: an invalid topic enum cost an embed). ab.sh now validates evals//PROMPT.md through the harness's own parseEvalMarkdown before any paid sync; skipped under the AB_EVAL_CMD test hook. Probed live: missing eval and the exact invalid-topic case both refuse pre-spend. --- workspace/scripts/ab.sh | 27 +++++++++++++++++++++++---- workspace/scripts/ab.test.sh | 11 ++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh index ad5a7bf6..6f3c0fa6 100755 --- a/workspace/scripts/ab.sh +++ b/workspace/scripts/ab.sh @@ -77,15 +77,34 @@ fi : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY not in keychain — see README}" # A previous run killed mid-baseline (SIGKILL beats any trap) leaves the edit # in a stash marked with AB_STASH_MSG. Detect it BEFORE the no-unstaged-edit -# check below turns that into a cryptic "nothing to A/B". +# check below turns that into a cryptic "nothing to A/B". Print the EXACT +# stash ref: a plain `stash pop` takes the newest stash, which may be the +# user's own unrelated work sitting above the stranded one. AB_STASH_MSG="eval-workspace ab baseline stash" -if git -C "$CLONE" stash list | grep -F "$AB_STASH_MSG" >/dev/null; then +# `|| :` guards set -e/pipefail: no match (the normal case) exits grep nonzero +_stranded=$(git -C "$CLONE" stash list --format='%gd %gs' | grep -F "$AB_STASH_MSG" | awk '{print $1}' | sort -t'{' -k2 -rn) || _stranded="" +if [ -n "$_stranded" ]; then echo "a previous A/B was interrupted mid-baseline — your edit is stranded in a stash." >&2 - echo "recover it, then re-run:" >&2 - echo " git -C $CLONE stash pop" >&2 + echo "recover it (exact ref; highest index first so the others keep their refs), then re-run:" >&2 + for _ref in $_stranded; do + echo " git -C $CLONE stash pop '$_ref'" >&2 + done echo " mise run ab $EVAL ${PATHS[0]} # (docs edits: the pop leaves the index at baseline until the next run re-embeds)" >&2 exit 1 fi +unset _stranded + +# Zero-cost eval validation BEFORE any paid sync: a schema/typo error used to +# surface only at harness discovery — after the treatment re-embed had spent. +# Skipped under the AB_EVAL_CMD test hook (no real eval dir exists there). +if [ -z "${AB_EVAL_CMD:-}" ]; then + [ -f "evals/$EVAL/PROMPT.md" ] || { echo "no eval at evals/$EVAL (PROMPT.md missing)" >&2; exit 1; } + ( cd apps/framework && exec pnpm exec tsx -e " + import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; + import { readFileSync } from 'node:fs'; + parseEvalMarkdown(readFileSync('../../evals/' + process.argv[1] + '/PROMPT.md', 'utf8'), 'evals/' + process.argv[1] + '/PROMPT.md'); + " "$EVAL" ) || { echo "eval metadata invalid — fix evals/$EVAL/PROMPT.md before spending on runs" >&2; exit 1; } +fi # A/B reverts tracked, unstaged working-tree edits only, via a scoped git stash. # `stash push -- ` merges whole-index state, so ANY staged entry in the # clone (even an unrelated intent-to-add) breaks it. Require a clean index. diff --git a/workspace/scripts/ab.test.sh b/workspace/scripts/ab.test.sh index 85686286..6b8851f3 100755 --- a/workspace/scripts/ab.test.sh +++ b/workspace/scripts/ab.test.sh @@ -51,13 +51,18 @@ edited=$(git -C "$SK" hash-object "$REL") index0=$(git -C "$SK" diff --cached -- "$REL") # empty (was clean) # --- a stranded baseline stash (SIGKILL mid-run) must be detected pre-spend -# with the recovery command, BEFORE the no-unstaged-edit check muddies it --- +# with the EXACT stash ref — a user's own newer stash sits above it, and a +# plain `stash pop` would take that instead --- git -C "$SK" stash push -q -m "eval-workspace ab baseline stash" -- "$REL" +printf '\n\n' >> "$FILE" +git -C "$SK" stash push -q -m "user's own unrelated stash" -- "$REL" # newer: stash@{0}; stranded: stash@{1} out0=$(ANTHROPIC_API_KEY=dummy bash workspace/scripts/ab.sh "$EVAL" "$EXP" "$FILE" 2>&1; echo "rc=$?") ck "stranded stash refused" "$(printf '%s' "$out0" | grep -c 'interrupted mid-baseline')" "1" -ck "stranded stash names recovery" "$(printf '%s' "$out0" | grep -c "git -C $SK stash pop")" "1" +ck "recovery names the exact ref" "$(printf '%s' "$out0" | grep -cF "git -C $SK stash pop 'stash@{1}'")" "1" +ck "recovery ignores the decoy" "$(printf '%s' "$out0" | grep -cF "stash@{0}")" "0" ck "stranded stash exits nonzero" "$(printf '%s' "$out0" | grep -c 'rc=1')" "1" -git -C "$SK" stash pop -q # put the edit back for the tests below +git -C "$SK" stash drop -q 'stash@{0}' # discard the decoy +git -C "$SK" stash pop -q # put the edit back for the tests below # observable sync hook: every sync appends a line (treatment, baseline, restore = 3) CNT=/tmp/ab_selftest_sync.cnt From ce5f869fda04daba8350f365ec33940238ca0843 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Fri, 24 Jul 2026 17:16:40 +0200 Subject: [PATCH 13/18] fix: mise ab wrapper refuses a path in the experiment slot 'mise run ab ' silently fed p2 as the experiment name, which would only fail at harness discovery - after the paid docs sync. Refuse pre-spend with the direct-script form for multiple paths. --- workspace/scripts/ab-task.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/workspace/scripts/ab-task.sh b/workspace/scripts/ab-task.sh index 7861cc41..50cff8e0 100755 --- a/workspace/scripts/ab-task.sh +++ b/workspace/scripts/ab-task.sh @@ -8,4 +8,12 @@ cd "$(dirname "$0")/../.." [ $# -gt 0 ] || exec workspace/scripts/ab-ready.sh [ $# -ge 2 ] || { echo "usage: mise run ab [experiment] (no args = readiness probe)" >&2; exit 2; } +# A path in $3 means the caller tried multiple edited files — $3 is the +# EXPERIMENT slot here, and a path there would only fail after the paid sync. +if [ $# -ge 3 ] && [ -e "$3" ]; then + echo "'$3' looks like an edited path, but the third argument is the experiment." >&2 + echo "for multiple edited files call the script directly:" >&2 + echo " workspace/scripts/ab.sh $1 $2 $3${4:+ ...}" >&2 + exit 2 +fi exec workspace/scripts/ab.sh "$1" "${3:-claude-sonnet-5}" "$2" From 1c3b14e2cf9c1f24fe23ef08a67a0a6391e07d8d Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Sat, 25 Jul 2026 11:30:54 +0200 Subject: [PATCH 14/18] feat: vs-main screen + per-worktree docs isolation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vs-main — treatment-only screen against published results (evals main): mise run vs-main … [--experiment ] [--runs N] Runs eval(s) in the edited world (mcp/docs/skills edits combined; dirty trees auto-detected and synced) and diffs against the freshest bot-refreshed row in apps/web/src/data/{regression-,}eval-results.json. No baseline arm, no stashes, no git mutation. Receipts record the published arm's provenance (result commit + parent SHA + age + attempts) and the treatment's usual provenance embed. Refuses pre-spend on unknown eval/experiment (lists the published alternatives) and reuses ab.sh's zero-cost eval-metadata gate. A published-vs-local flip is a screen, not causal proof; the output says so and points at mise run ab. docs-isolate — per-worktree docs stacks for interference-free sessions: mise run docs-isolate && mise run docs-up && workspace/scripts/docs-copy-index.sh Writes an UNTRACKED overlay workdir (workspace/.docs-stack: rewritten config.toml + symlinks into the submodule) so no tracked file in any clone changes. Slots (project id + 55[1-9]21 port block + docs-api 300[1-9]) are allocated through a machine-shared registry under an atomic mkdir lock, keyed by canonical worktree path — concurrent isolations cannot collide (a docker-only scan would race: containers appear at docs-up, not isolate). docs-copy-index seeds the new stack from a sibling's warm index via pg_dump/restore: zero OpenAI spend, checksums included. All docs scripts + ab preflights now resolve the stack through docs-profile.sh (workdir, project id, container, ports) instead of hardcoded names/ports; the primary worktree keeps its exact current identity (slot 0). Self-tests: vs-main.test 13/0, ab.test 28/0. --- .gitignore | 2 + mise.toml | 12 ++ workspace/README.md | 29 +++++ workspace/scripts/ab-demo.sh | 7 +- workspace/scripts/ab-ready.sh | 10 +- workspace/scripts/ab.sh | 6 +- workspace/scripts/docs-api.sh | 7 +- workspace/scripts/docs-copy-index.sh | 35 ++++++ workspace/scripts/docs-down.sh | 3 +- workspace/scripts/docs-embed-env.sh | 3 +- workspace/scripts/docs-isolate.sh | 84 ++++++++++++++ workspace/scripts/docs-profile.sh | 24 ++++ workspace/scripts/docs-up.sh | 5 +- workspace/scripts/vs-main.sh | 163 +++++++++++++++++++++++++++ workspace/scripts/vs-main.test.sh | 44 ++++++++ 15 files changed, 418 insertions(+), 16 deletions(-) create mode 100755 workspace/scripts/docs-copy-index.sh create mode 100755 workspace/scripts/docs-isolate.sh create mode 100644 workspace/scripts/docs-profile.sh create mode 100755 workspace/scripts/vs-main.sh create mode 100755 workspace/scripts/vs-main.test.sh diff --git a/.gitignore b/.gitignore index 20ebb2ca..7d5d38c5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,7 @@ results/*/ # eval-source workspace glue (workspace/README.md) /results-ab/ +/results-vs-main/ /.docs-index-stamp.json +/workspace/.docs-stack/ /.publish/ diff --git a/mise.toml b/mise.toml index 37ff8f10..2d6f2093 100644 --- a/mise.toml +++ b/mise.toml @@ -48,6 +48,18 @@ run = "workspace/scripts/ab-demo.sh" description = "Zero-cost self-test of the A/B runner (fakes the eval; only touches a clean skill file)" run = "workspace/scripts/ab.test.sh" +[tasks.vs-main] +description = "Screen … against the latest published result on main — treatment only, no stashes, multi-session safe (flags: --experiment, --runs)" +run = "workspace/scripts/vs-main.sh" + +[tasks.vs-main-test] +description = "Zero-cost self-test of the vs-main screen (fakes the eval run; reads real published baselines)" +run = "workspace/scripts/vs-main.test.sh" + +[tasks.docs-isolate] +description = "Give THIS worktree its own docs stack (project id + ports) for interference-free parallel sessions" +run = "workspace/scripts/docs-isolate.sh" + [tasks.status-test] description = "Self-test of status.sh's Next/Ready diagnosis against synthetic workspace states" run = "workspace/scripts/status.test.sh" diff --git a/workspace/README.md b/workspace/README.md index f5c217f1..ed1cc401 100644 --- a/workspace/README.md +++ b/workspace/README.md @@ -68,6 +68,35 @@ loop (spend-gated, asks first). Writing your own discriminator eval? Ask for one precise, docs-only fact ("name the exact package for X") — vague questions make both arms search for minutes before converging. +## Screening against published results (`vs-main`) + +The cheap iteration loop: run eval(s) in YOUR edited world (any mix of +mcp/docs/skills edits) and diff against the latest published result on evals +`main` — no baseline arm, no stashes, no git mutation anywhere. + +```bash +mise run vs-main […] [--experiment ] [--runs N] +``` + +Dirty submodule trees are detected and synced automatically (mcp build, docs +re-embed); receipts land in `results-vs-main/*.json` with the published arm's +result commit + parent SHA and age. The published arm ran in the scheduled CI +world (published mcp package, prod docs index, model state at refresh time), +so a flip is a **screen** — confirm causal claims with one paired `mise run +ab`. Free self-test: `mise run vs-main-test`. + +## Parallel sessions (worktrees) + +MCP/skills edits are per-worktree by construction (tools-mode evals only — +CLI/local-stack evals share host ports, one at a time machine-wide). The docs +stack is shared by default; give a worktree its own (project id + port block ++ docs-api port, allocated through a locked machine-shared registry): + +```bash +mise run docs-isolate # untracked overlay; no clone file touched +mise run docs-up && workspace/scripts/docs-copy-index.sh # free seed from a sibling stack +``` + ## Patches & publishing Local changes to the patched repos (the `submodules/supabase` and diff --git a/workspace/scripts/ab-demo.sh b/workspace/scripts/ab-demo.sh index 8562ef71..0a65a8e1 100755 --- a/workspace/scripts/ab-demo.sh +++ b/workspace/scripts/ab-demo.sh @@ -32,10 +32,11 @@ done [ -e submodules/supabase/.git ] || fail "supabase not cloned" [ -n "${ANTHROPIC_API_KEY:-}" ] || fail "ANTHROPIC_API_KEY missing" [ -n "${OPENAI_API_KEY:-}" ] || fail "OPENAI_API_KEY missing" -docker exec supabase_db_eval-workspace-content true 2>/dev/null || fail "content DB not running" -pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) +source workspace/scripts/docs-profile.sh +docker exec "$CONTENT_DB_CONTAINER" true 2>/dev/null || fail "content DB not running" +pages=$(docker exec "$CONTENT_DB_CONTAINER" psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) [ "${pages:-0}" -gt 0 ] 2>/dev/null || fail "docs index not seeded" -curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql || fail "docs-api not serving on :3001" +curl -sf -o /dev/null "$CONTENT_URL" || fail "docs-api not serving on :$DOCS_API_PORT" git -C submodules/supabase diff --quiet -- "${GUIDE#submodules/supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}" [ ! -e "$EVAL_DST" ] || fail "$EVAL_DST already exists — remove it first" diff --git a/workspace/scripts/ab-ready.sh b/workspace/scripts/ab-ready.sh index 5830865a..f32bb634 100755 --- a/workspace/scripts/ab-ready.sh +++ b/workspace/scripts/ab-ready.sh @@ -27,15 +27,17 @@ fi echo echo "docs loop (edit submodules/supabase/apps/docs/content/… pages):" +source workspace/scripts/docs-profile.sh +[ "$CONTENT_WORKDIR" = submodules/supabase ] || ok "isolated worktree stack ($CONTENT_PROJECT_ID)" if [ -e submodules/supabase/.git ]; then ok "supabase (apps/docs) cloned"; else miss "supabase not cloned" "mise run clone-docs"; fi -if docker exec supabase_db_eval-workspace-content true 2>/dev/null; then +if docker exec "$CONTENT_DB_CONTAINER" true 2>/dev/null; then ok "content DB running" - pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) - if [ "${pages:-0}" -gt 0 ] 2>/dev/null; then ok "index seeded ($pages pages)"; else miss "index empty" "mise run docs-seed # one-time full embed (OpenAI \$; asks to confirm)"; fi + pages=$(docker exec "$CONTENT_DB_CONTAINER" psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) + if [ "${pages:-0}" -gt 0 ] 2>/dev/null; then ok "index seeded ($pages pages)"; else miss "index empty" "mise run docs-seed # one-time full embed (OpenAI \$; asks to confirm) — or free from a sibling stack: workspace/scripts/docs-copy-index.sh"; fi else miss "content DB not running" "mise run docs-up" fi -if curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql; then ok "docs-api serving :3001"; else miss "docs-api not serving" "mise run docs-api # keep it running in a separate terminal"; fi +if curl -sf -o /dev/null "$CONTENT_URL"; then ok "docs-api serving :$DOCS_API_PORT"; else miss "docs-api not serving" "mise run docs-api # keep it running in a separate terminal"; fi if have_key OPENAI_API_KEY; then ok "OPENAI_API_KEY present"; else miss "OPENAI_API_KEY missing" "mise run store-key OPENAI_API_KEY"; fi echo diff --git a/workspace/scripts/ab.sh b/workspace/scripts/ab.sh index 6f3c0fa6..b476734c 100755 --- a/workspace/scripts/ab.sh +++ b/workspace/scripts/ab.sh @@ -51,7 +51,7 @@ for p in "${PATHS[@]}"; do done MCP="$PWD/submodules/mcp/packages/mcp-server-supabase" -CONTENT_URL="http://127.0.0.1:3001/docs/api/graphql" # also in affected.ts / docs-api.sh / workspace README — keep in sync +source workspace/scripts/docs-profile.sh # CONTENT_URL, CONTENT_DB_CONTAINER (per-worktree; also in docs-api.sh / affected.ts) RUN_ENV=() case "$LOOP" in docs) RUN_ENV=( "SUPABASE_MCP_SERVER_PATH=$MCP" "SUPABASE_CONTENT_API_URL=$CONTENT_URL" ) ;; @@ -121,10 +121,10 @@ done if [ "$LOOP" = docs ]; then : "${OPENAI_API_KEY:?OPENAI_API_KEY not in keychain — the docs re-embed needs it}" - curl -sf -o /dev/null "$CONTENT_URL" || { echo "docs-api not reachable on :3001 — run \`mise run docs-api\` in another terminal first" >&2; exit 1; } + curl -sf -o /dev/null "$CONTENT_URL" || { echo "docs-api not reachable on :$DOCS_API_PORT — run \`mise run docs-api\` in another terminal first" >&2; exit 1; } # Refuse BEFORE spending: an empty-but-running DB would otherwise turn the # treatment sync into a full paid seed instead of an incremental re-embed. - pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) + pages=$(docker exec "$CONTENT_DB_CONTAINER" psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) [ "${pages:-0}" -gt 0 ] 2>/dev/null || { echo "docs index is not seeded (page count: ${pages:-0}) — run \`mise run docs-seed\` once first (mise run ab with no args = full readiness probe)" >&2; exit 1; } [ -d "$MCP/dist" ] || { echo "building local mcp (needed for search_docs routing)…"; ( cd submodules/mcp && pnpm install && pnpm build ); } fi diff --git a/workspace/scripts/docs-api.sh b/workspace/scripts/docs-api.sh index a3e6fb0c..f99b6b30 100755 --- a/workspace/scripts/docs-api.sh +++ b/workspace/scripts/docs-api.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash # Serve the docs content GraphQL API locally (standalone adapter) for search_docs. -# Point evals at it: SUPABASE_CONTENT_API_URL=http://127.0.0.1:3001/docs/api/graphql +# Port + stack come from this worktree's docs profile (docs-profile.sh); +# point evals at it: SUPABASE_CONTENT_API_URL=$CONTENT_URL set -euo pipefail cd "$(dirname "$0")/../.." +source workspace/scripts/docs-profile.sh source workspace/scripts/load-keys.sh set -a source submodules/supabase/apps/docs/.env.development set +a -eval "$(supabase status --workdir submodules/supabase -o env)" +eval "$(supabase status --workdir "$CONTENT_WORKDIR" -o env)" : "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" # Runs the locally installed tsx binary directly (no `pnpm exec`: NODE_OPTIONS # must reach only the server process — pnpm's own node chokes on the loader @@ -18,6 +20,7 @@ eval "$(supabase status --workdir submodules/supabase -o env)" STUB_REGISTER=$PWD/workspace/scripts/sentry-stub-register.mjs cd submodules/supabase/apps/docs [ -x node_modules/.bin/tsx ] || { echo "tsx not installed — run: mise run clone-docs" >&2; exit 1; } +PORT="$DOCS_API_PORT" \ NODE_ENV=development \ NEXT_PUBLIC_SUPABASE_URL="$API_URL" \ NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" \ diff --git a/workspace/scripts/docs-copy-index.sh b/workspace/scripts/docs-copy-index.sh new file mode 100755 index 00000000..fd0915ba --- /dev/null +++ b/workspace/scripts/docs-copy-index.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copy a seeded docs index (page + page_section, embeddings + checksums) from +# another local stack's DB container into THIS worktree's. Zero OpenAI spend: +# the checksums come along, so the next docs-index re-embeds only your edits. +# +# usage: workspace/scripts/docs-copy-index.sh [source-container] +# default source: the primary worktree's supabase_db_eval-workspace-content +set -euo pipefail +cd "$(dirname "$0")/../.." +source workspace/scripts/docs-profile.sh + +src="${1:-supabase_db_eval-workspace-content}" +dst="$CONTENT_DB_CONTAINER" +[ "$src" != "$dst" ] || { echo "source and destination are the same container ($dst) — pass a sibling's container name" >&2; exit 1; } +docker exec "$src" true 2>/dev/null || { echo "source container not running: $src (docs-up in that worktree first, or pass another source)" >&2; exit 1; } +docker exec "$dst" true 2>/dev/null || { echo "this worktree's stack is not running: $dst — run: mise run docs-up" >&2; exit 1; } + +pages=$(docker exec "$src" psql -U postgres -d postgres -tAc 'select count(*) from public.page') +[ "${pages:-0}" -gt 0 ] || { echo "source index is empty ($src) — nothing to copy" >&2; exit 1; } + +# dump to a temp file (a straight pipe would hide a mid-stream pg_dump failure) +tmp=$(mktemp /tmp/docs-index-copy.XXXXXX.sql) +trap 'rm -f "$tmp"' EXIT +docker exec "$src" pg_dump -U postgres -d postgres --data-only -t public.page -t public.page_section > "$tmp" + +docker exec "$dst" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q \ + -c 'TRUNCATE public.page, public.page_section RESTART IDENTITY CASCADE' +docker exec -i "$dst" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q < "$tmp" +docker exec "$dst" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q -c " + select setval(pg_get_serial_sequence('public.page','id'), coalesce(max(id),1)) from public.page; + select setval(pg_get_serial_sequence('public.page_section','id'), coalesce(max(id),1)) from public.page_section;" + +got=$(docker exec "$dst" psql -U postgres -d postgres -tAc 'select count(*) from public.page') +[ "$got" = "$pages" ] || { echo "copy incomplete: source $pages pages, destination $got" >&2; exit 1; } +echo "copied $got pages -> $dst (\$0.00 — the next docs-index re-embeds only pages your tree changed)" diff --git a/workspace/scripts/docs-down.sh b/workspace/scripts/docs-down.sh index 2590b9e4..c99241ff 100755 --- a/workspace/scripts/docs-down.sh +++ b/workspace/scripts/docs-down.sh @@ -2,4 +2,5 @@ set -euo pipefail cd "$(dirname "$0")/../.." -supabase stop --workdir submodules/supabase +source workspace/scripts/docs-profile.sh +supabase stop --workdir "$CONTENT_WORKDIR" diff --git a/workspace/scripts/docs-embed-env.sh b/workspace/scripts/docs-embed-env.sh index e347aa2a..27bf2123 100644 --- a/workspace/scripts/docs-embed-env.sh +++ b/workspace/scripts/docs-embed-env.sh @@ -20,7 +20,8 @@ source workspace/scripts/load-keys.sh set -a source submodules/supabase/apps/docs/.env.development set +a -eval "$(supabase status --workdir submodules/supabase -o env)" +source workspace/scripts/docs-profile.sh +eval "$(supabase status --workdir "$CONTENT_WORKDIR" -o env)" export NEXT_PUBLIC_SUPABASE_URL="$API_URL" export NEXT_PUBLIC_SUPABASE_ANON_KEY="$PUBLISHABLE_KEY" export SUPABASE_SECRET_KEY="$SECRET_KEY" diff --git a/workspace/scripts/docs-isolate.sh b/workspace/scripts/docs-isolate.sh new file mode 100755 index 00000000..556dafdc --- /dev/null +++ b/workspace/scripts/docs-isolate.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Give THIS worktree its own docs stack (project id + port block + docs-api +# port) so docs runs in parallel sessions never share a DB or API. +# +# usage: workspace/scripts/docs-isolate.sh [slot] # slot 1-9; default: allocate +# +# Slot allocation goes through a machine-shared registry +# (~/.local/state/eval-workspace/docs-slots.json) under an atomic mkdir lock, +# keyed by canonical worktree path — two sessions isolating concurrently can +# never pick the same slot (a docker-only scan would race: the container +# appears at docs-up, not at isolate). Re-running returns your existing slot; +# entries for deleted worktrees are pruned on allocation. +# +# Writes an UNTRACKED overlay workdir (workspace/.docs-stack, git-ignored): +# a rewritten config.toml plus symlinks to the submodule's migrations/seed/ +# functions/buckets. No tracked file in any clone is touched — your in-flight +# work stays byte-for-byte untouched. Every docs script picks the overlay up +# automatically (docs-profile.sh). Symlinks track the submodule live, so +# patch/content updates flow through without regeneration. +# +# After isolating, seed this stack for free from a sibling's warm index: +# mise run docs-up && workspace/scripts/docs-copy-index.sh +set -euo pipefail +cd "$(dirname "$0")/../.." + +src=submodules/supabase/supabase +[ -f "$src/config.toml" ] || { echo "no docs submodule config at $src — run: mise run clone-docs" >&2; exit 1; } + +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/eval-workspace" +REG="$STATE_DIR/docs-slots.json" +LOCK="$STATE_DIR/docs-slots.lock" +mkdir -p "$STATE_DIR" + +# atomic mkdir lock (no flock on stock macOS); stale after 60s = crashed holder +for _i in $(seq 1 60); do + mkdir "$LOCK" 2>/dev/null && break + [ "$_i" = 60 ] && { echo "slot registry locked for 60s ($LOCK) — remove it if no other isolate is running" >&2; exit 1; } + sleep 1 +done +trap 'rmdir "$LOCK" 2>/dev/null || true' EXIT + +slot=$(WANT="${1:-}" REG="$REG" WT="$(pwd -P)" node -e ' +const fs = require("fs"); +const { WANT, REG, WT } = process.env; +let reg = {}; +try { reg = JSON.parse(fs.readFileSync(REG, "utf8")); } catch {} +// prune slots whose worktree no longer exists +for (const [s, p] of Object.entries(reg)) if (!fs.existsSync(p)) delete reg[s]; +const mine = Object.entries(reg).find(([, p]) => p === WT)?.[0]; +let slot; +if (WANT) { + if (!/^[1-9]$/.test(WANT)) { console.error("slot must be 1-9 (slot 0 = the primary worktree default)"); process.exit(2); } + if (reg[WANT] && reg[WANT] !== WT) { console.error(`slot ${WANT} is taken by ${reg[WANT]}`); process.exit(1); } + if (mine && mine !== WANT) delete reg[mine]; + slot = WANT; +} else if (mine) { + slot = mine; +} else { + slot = [1,2,3,4,5,6,7,8,9].find((s) => !reg[s]); + if (!slot) { console.error("all 9 slots taken: " + JSON.stringify(reg, null, 1)); process.exit(1); } +} +reg[slot] = WT; +fs.writeFileSync(REG, JSON.stringify(reg, null, 1) + "\n"); +console.log(String(slot)); +') + +ov=workspace/.docs-stack/supabase +mkdir -p "$ov" +for f in buckets functions migrations seed.sql; do + ln -sfn "../../../submodules/supabase/supabase/$f" "$ov/$f" +done + +SLOT="$slot" node -e ' +const fs = require("fs"); +const slot = Number(process.env.SLOT); +let t = fs.readFileSync(process.argv[1], "utf8"); +t = t.replace(/^project_id = ".*"$/m, `project_id = "eval-workspace-content-${slot}"`); +t = t.replace(/^port = 55[0-9]([0-9][0-9])$/gm, (_, tail) => `port = ${55300 + 100 * slot + Number(tail)}`); +fs.writeFileSync(process.argv[2], t); +' "$src/config.toml" "$ov/config.toml" + +source workspace/scripts/docs-profile.sh +echo "isolated (slot $slot): workdir=$CONTENT_WORKDIR project_id=$CONTENT_PROJECT_ID api=$CONTENT_API_PORT docs-api=$DOCS_API_PORT" +echo "next: mise run docs-up && workspace/scripts/docs-copy-index.sh # free seed from a sibling stack" diff --git a/workspace/scripts/docs-profile.sh b/workspace/scripts/docs-profile.sh new file mode 100644 index 00000000..64779dbd --- /dev/null +++ b/workspace/scripts/docs-profile.sh @@ -0,0 +1,24 @@ +# Per-worktree docs stack identity — source me from the repo root. +# The primary worktree uses the docs submodule itself as the supabase workdir +# (config.toml pinned by the ports patch: slot 0). An isolated worktree (see +# docs-isolate.sh) carries an UNTRACKED overlay workdir with its own project +# id + port block, so its containers, DB, and docs-api never touch another +# session's — and no user tree is ever modified. +# Sets: CONTENT_WORKDIR, CONTENT_PROJECT_ID, CONTENT_DB_CONTAINER, +# CONTENT_API_PORT, DOCS_API_PORT, CONTENT_URL +if [ -f workspace/.docs-stack/supabase/config.toml ]; then + CONTENT_WORKDIR=workspace/.docs-stack +else + CONTENT_WORKDIR=submodules/supabase +fi +_cfg="$CONTENT_WORKDIR/supabase/config.toml" +CONTENT_PROJECT_ID=$(sed -n 's/^project_id = "\(.*\)"$/\1/p' "$_cfg" 2>/dev/null | head -1) +CONTENT_API_PORT=$(sed -n 's/^port = \(55[0-9][0-9][0-9]\)$/\1/p' "$_cfg" 2>/dev/null | head -1) +# fall back to the primary defaults when the submodule isn't cloned yet +CONTENT_PROJECT_ID="${CONTENT_PROJECT_ID:-eval-workspace-content}" +CONTENT_API_PORT="${CONTENT_API_PORT:-55321}" +CONTENT_DB_CONTAINER="supabase_db_${CONTENT_PROJECT_ID}" +# slot 0 (primary, 55321) -> 3001; slot k (55321+100k) -> 3001+k +DOCS_API_PORT=$(( 3001 + (CONTENT_API_PORT - 55321) / 100 )) +CONTENT_URL="http://127.0.0.1:${DOCS_API_PORT}/docs/api/graphql" +unset _cfg diff --git a/workspace/scripts/docs-up.sh b/workspace/scripts/docs-up.sh index 87f270c7..38031fcb 100755 --- a/workspace/scripts/docs-up.sh +++ b/workspace/scripts/docs-up.sh @@ -2,12 +2,13 @@ set -euo pipefail cd "$(dirname "$0")/../.." -supabase start --workdir submodules/supabase \ +source workspace/scripts/docs-profile.sh +supabase start --workdir "$CONTENT_WORKDIR" \ -x realtime,storage-api,imgproxy,mailpit,postgres-meta,studio,edge-runtime,logflare,vector,supavisor # Upstream page migrations grant service_role only Dxt (no CRUD). The new secret # key authenticates as service_role and bypasses RLS but NOT SQL GRANTs, so the # embedder (service_role) can't write the content tables. Grant CRUD locally # (idempotent; native psql is absent on this host, so run it in the db container). -docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -q -c \ +docker exec "$CONTENT_DB_CONTAINER" psql -U postgres -d postgres -q -c \ "GRANT ALL ON public.page, public.page_section TO service_role; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO service_role; GRANT SELECT ON public.page, public.page_section TO anon, authenticated;" diff --git a/workspace/scripts/vs-main.sh b/workspace/scripts/vs-main.sh new file mode 100755 index 00000000..8ec42d93 --- /dev/null +++ b/workspace/scripts/vs-main.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# Treatment-only screen: run eval(s) in YOUR edited world and diff against the +# latest published result on evals main (the bot refreshes of +# apps/web/src/data/{regression-,}eval-results.json). +# +# No baseline arm, no stashes, no git mutation anywhere — safe to run from +# multiple sessions/worktrees at once. Docs edits additionally want this +# worktree's own stack (workspace/scripts/docs-isolate.sh) when other sessions +# also run docs; mcp/skills edits are per-worktree by construction. +# +# The published arm ran in the scheduled CI world (published mcp package, prod +# docs index, model state at refresh time). A flip here is a SCREEN, not +# causal proof — confirm with a paired A/B (mise run ab) before claiming +# causality. The receipt records the exact published provenance available: +# result commit, its parent (the harness revision the scheduled run merged +# onto), commit time, experiment, and attempts. +# +# usage: workspace/scripts/vs-main.sh [...] [--experiment ] [--runs N] +# default experiment: claude-code-sonnet-5 (the refreshed suites' flagship) +# default runs: the published row's attempts (usually 1) +# +# VSMAIN_EVAL_CMD / VSMAIN_NO_FETCH / VSMAIN_SKIP_SYNC: test hooks (vs-main.test.sh). +set -euo pipefail +cd "$(dirname "$0")/../.." + +EXP=claude-code-sonnet-5; RUNS=""; EVALS=() +while [ $# -gt 0 ]; do + case "$1" in + --experiment) EXP="${2:?--experiment needs a value}"; shift 2 ;; + --runs) RUNS="${2:?--runs needs a value}"; shift 2 ;; + -*) echo "unknown flag: $1" >&2; exit 2 ;; + *) EVALS+=("$1"); shift ;; + esac +done +[ ${#EVALS[@]} -gt 0 ] || { echo "usage: workspace/scripts/vs-main.sh [...] [--experiment ] [--runs N]" >&2; exit 2; } + +for k in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do + v="$(security find-generic-password -a "$USER" -s "eval-workspace:$k" -w 2>/dev/null || true)" + [ -n "$v" ] && export "$k=$v" +done + +OUT=results-vs-main; mkdir -p "$OUT" + +# --- resolve published baselines (free; refuses before any spend) --- +[ -n "${VSMAIN_NO_FETCH:-}" ] || git fetch -q origin main +EXP="$EXP" OUT="$OUT" node - "${EVALS[@]}" <<'EOF' +const fs = require("fs"); +const { execFileSync } = require("child_process"); +const { EXP, OUT } = process.env; +const evals = process.argv.slice(2); +const files = [ + "apps/web/src/data/regression-eval-results.json", + "apps/web/src/data/eval-results.json", +]; +const best = {}; // eval -> {row, meta} freshest matching row +const seen = {}; // eval -> Set(experiments) for the refusal message +for (const f of files) { + let rows; + try { rows = JSON.parse(execFileSync("git", ["show", `origin/main:${f}`], { maxBuffer: 1 << 28 }).toString()); } + catch { continue; } + const [commit, parent, committedAt] = + execFileSync("git", ["log", "origin/main", "-1", "--format=%H %P %cI", "--", f]).toString().trim().split(" "); + for (const r of rows) { + if (!evals.includes(r.eval)) continue; + (seen[r.eval] ??= new Set()).add(r.experiment); + if (r.experiment !== EXP) continue; + const meta = { file: f, commit, parent, committedAt }; + if (!best[r.eval] || new Date(committedAt) > new Date(best[r.eval].meta.committedAt)) + best[r.eval] = { row: r, meta }; + } +} +let missing = false; +for (const e of evals) { + if (!best[e]) { + missing = true; + const alts = [...(seen[e] ?? [])]; + console.error(alts.length + ? `no published ${EXP} result for ${e} on origin/main (published experiments: ${alts.join(", ")})` + : `no published result for ${e} on origin/main at all — vs-main needs a published baseline; use mise run ab`); + continue; + } + const { row, meta } = best[e]; + fs.writeFileSync(`${OUT}/${e}.published.json`, JSON.stringify({ ...row, vsMainBaseline: meta }, null, 1) + "\n"); +} +process.exit(missing ? 1 : 0); +EOF + +# --- zero-cost eval validation (same gate as ab.sh) --- +if [ -z "${VSMAIN_EVAL_CMD:-}" ]; then + : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY not in keychain — see README}" + for e in "${EVALS[@]}"; do + [ -f "evals/$e/PROMPT.md" ] || { echo "no eval at evals/$e (PROMPT.md missing)" >&2; exit 1; } + ( cd apps/framework && exec pnpm exec tsx -e " + import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; + import { readFileSync } from 'node:fs'; + parseEvalMarkdown(readFileSync('../../evals/' + process.argv[1] + '/PROMPT.md', 'utf8'), 'evals/' + process.argv[1] + '/PROMPT.md'); + " "$e" ) || { echo "eval metadata invalid — fix evals/$e/PROMPT.md before spending on runs" >&2; exit 1; } + done +fi + +# --- sync YOUR world, treatment only: no reverts, nothing to restore --- +MCP="$PWD/submodules/mcp/packages/mcp-server-supabase" +source workspace/scripts/docs-profile.sh +RUN_ENV=(); DIRTY=() +if [ -z "${VSMAIN_SKIP_SYNC:-}" ]; then + if [ -e submodules/mcp/.git ] && [ -n "$(git -C submodules/mcp status --porcelain)" ]; then + DIRTY+=(mcp) + echo "== mcp tree dirty: building local server ==" + ( cd submodules/mcp && pnpm build ) + RUN_ENV+=( "SUPABASE_MCP_SERVER_PATH=$MCP" ) + fi + if [ -e submodules/supabase/.git ] && [ -n "$(git -C submodules/supabase status --porcelain -- apps/docs/content)" ]; then + DIRTY+=(docs) + : "${OPENAI_API_KEY:?OPENAI_API_KEY not in keychain — the docs re-embed needs it}" + curl -sf -o /dev/null "$CONTENT_URL" || { echo "docs-api not reachable on :$DOCS_API_PORT — run \`mise run docs-api\` in another terminal first (isolated worktrees: workspace/scripts/docs-isolate.sh, then docs-up + docs-copy-index)" >&2; exit 1; } + pages=$(docker exec "$CONTENT_DB_CONTAINER" psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0) + [ "${pages:-0}" -gt 0 ] 2>/dev/null || { echo "docs index is not seeded (page count: ${pages:-0}) — mise run docs-seed, or free from a sibling: workspace/scripts/docs-copy-index.sh" >&2; exit 1; } + [ -d "$MCP/dist" ] || { echo "building local mcp (needed for search_docs routing)…"; ( cd submodules/mcp && pnpm install && pnpm build ); } + echo "== docs tree dirty: re-embedding changed pages ==" + workspace/scripts/docs-index.sh + RUN_ENV+=( "SUPABASE_MCP_SERVER_PATH=$MCP" "SUPABASE_CONTENT_API_URL=$CONTENT_URL" ) + fi + if [ -e submodules/agent-skills/.git ] && [ -n "$(git -C submodules/agent-skills status --porcelain)" ]; then + DIRTY+=(skills) # read live via symlink; no sync step + fi + [ ${#DIRTY[@]} -gt 0 ] || echo "note: no local edits detected in submodules/{mcp,supabase/apps/docs/content,agent-skills} — this compares your pinned world against published" +fi + +# --- run treatment + report, one eval at a time --- +FAILED=0 +for e in "${EVALS[@]}"; do + runs="$RUNS" + [ -n "$runs" ] || runs=$(node -pe 'require(`./${process.env.OUT}/${process.argv[1]}.published.json`).attempts || 1' "$e" 2>/dev/null || echo 1) + RES="results/$EXP/$e.json" + echo "== treatment: $e ($EXP, runs=$runs, edits: ${DIRTY[*]:-none}) ==" + if [ -n "${VSMAIN_EVAL_CMD:-}" ]; then + RES="$RES" EVAL="$e" bash -c "$VSMAIN_EVAL_CMD" + else + env ${RUN_ENV[@]+"${RUN_ENV[@]}"} pnpm eval --eval "$e" --experiment "$EXP" --runs "$runs" || { echo "eval run failed: $e" >&2; FAILED=1; continue; } + fi + [ -f "$RES" ] || { echo "no result at $RES — check the eval/experiment ids" >&2; FAILED=1; continue; } + cp "$RES" "$OUT/$e.treatment.json" + env ${RUN_ENV[@]+"${RUN_ENV[@]}"} node workspace/scripts/provenance.mjs --embed "$OUT/$e.treatment.json" + + EVAL="$e" EXP="$EXP" OUT="$OUT" DIRTY="${DIRTY[*]:-none}" node -e ' +const path=require("path"); +const f=(p)=>{try{return require(path.resolve(p))}catch{return null}}; +const {EVAL,EXP,OUT,DIRTY}=process.env; +const b=f(`${OUT}/${EVAL}.published.json`), t=f(`${OUT}/${EVAL}.treatment.json`); +const chk=(r)=>{const c=(r&&r.checks)||[];return `${c.filter(x=>x&&x.passed).length}/${c.length}`}; +const row=(l,r,extra)=>`${l.padEnd(10)} passed=${String(r&&r.passed).padEnd(5)} checks=${chk(r).padEnd(6)} docs.calls=${String(((r&&r.docs&&r.docs.calls)||[]).length).padEnd(3)} ${extra||""}`; +const m=(b&&b.vsMainBaseline)||{}; +const age=m.committedAt?Math.round((Date.now()-new Date(m.committedAt))/864e5):"?"; +console.log(`\n=== vs-main: ${EVAL} (${EXP}) ===`); +console.log(row("published",b,`main@${(m.commit||"").slice(0,7)} ${String(m.committedAt||"").slice(0,10)} (${age}d old, attempts ${b&&b.attempts})`)); +console.log(row("treatment",t,`your world (edits: ${DIRTY})`)); +const d=((t&&t.passed)?1:0)-((b&&b.passed)?1:0); +console.log(d>0?"-> IMPROVED vs published (FAIL->PASS)":d<0?"-> REGRESSED vs published (PASS->FAIL)":"-> no pass/fail change (compare checks / docs.calls)"); +console.log(`screen only: the published arm ran in the scheduled CI world — confirm causal claims with: mise run ab ${EVAL} `); +console.log(`saved: ${OUT}/${EVAL}.{published,treatment}.json`); +' +done +exit "$FAILED" diff --git a/workspace/scripts/vs-main.test.sh b/workspace/scripts/vs-main.test.sh new file mode 100755 index 00000000..b223ec83 --- /dev/null +++ b/workspace/scripts/vs-main.test.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Zero-cost self-test of vs-main.sh: fakes the eval run (VSMAIN_EVAL_CMD), +# reads REAL published baselines from origin/main (no fetch, no model spend, +# no docker). Requires origin/main to exist locally (any clone has it). +set -euo pipefail +cd "$(dirname "$0")/../.." + +pass=0; fail=0 +ck() { if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $1 (want[$3] got[$2])"; fi; } + +export VSMAIN_NO_FETCH=1 VSMAIN_SKIP_SYNC=1 + +# a real published eval id (regression suite; stable) — resolved dynamically so +# the test doesn't rot when the published set changes +EVAL=$(node -pe 'JSON.parse(require("child_process").execFileSync("git",["show","origin/main:apps/web/src/data/regression-eval-results.json"],{maxBuffer:1<<28})).find(r=>r.experiment==="claude-code-sonnet-5").eval') + +# --- unknown eval refused pre-spend, nonzero exit --- +out=$(bash workspace/scripts/vs-main.sh no-such-eval-xyz 2>&1; echo "rc=$?") +ck "unknown eval refused" "$(printf '%s' "$out" | grep -c 'no published result for no-such-eval-xyz')" "1" +ck "unknown eval exits nonzero" "$(printf '%s' "$out" | grep -c 'rc=1')" "1" + +# --- unknown experiment lists the published ones --- +out=$(bash workspace/scripts/vs-main.sh "$EVAL" --experiment bogus-model 2>&1; echo "rc=$?") +ck "unknown experiment refused" "$(printf '%s' "$out" | grep -c "no published bogus-model result for $EVAL")" "1" +ck "alternatives listed" "$(printf '%s' "$out" | grep -c 'published experiments:.*claude-code-sonnet-5')" "1" + +# --- happy path: fake eval run -> delta table + receipts --- +export VSMAIN_EVAL_CMD='mkdir -p "$(dirname "$RES")"; printf "{\"eval\":\"%s\",\"experiment\":\"claude-code-sonnet-5\",\"passed\":true,\"checks\":[{\"name\":\"x\",\"passed\":true}]}" "$EVAL" > "$RES"' +out=$(bash workspace/scripts/vs-main.sh "$EVAL" 2>&1; echo "rc=$?") +ck "delta table printed" "$(printf '%s' "$out" | grep -c "=== vs-main: $EVAL")" "1" +ck "published row shown" "$(printf '%s' "$out" | grep -c '^published .*main@')" "1" +ck "treatment row shown" "$(printf '%s' "$out" | grep -c '^treatment ')" "1" +ck "screen caveat present" "$(printf '%s' "$out" | grep -c 'screen only:')" "1" +ck "happy path exits zero" "$(printf '%s' "$out" | grep -c 'rc=0')" "1" +ck "published receipt has commit" "$(FORCE_COLOR=0 node -pe 'const b=require("./results-vs-main/"+process.argv[1]+".published.json"); /^[0-9a-f]{40}$/.test(b.vsMainBaseline.commit)?1:0' "$EVAL")" "1" +ck "treatment receipt has provenance" "$(FORCE_COLOR=0 node -pe 'require("./results-vs-main/"+process.argv[1]+".treatment.json").provenance?1:0' "$EVAL")" "1" +EVAL2=$(node -pe 'const rows=JSON.parse(require("child_process").execFileSync("git",["show","origin/main:apps/web/src/data/eval-results.json"],{maxBuffer:1<<28})); rows.filter(r=>r.experiment==="claude-code-sonnet-5").map(r=>r.eval).find(e=>e!==process.argv[1])' "$EVAL") +out=$(bash workspace/scripts/vs-main.sh "$EVAL" "$EVAL2" 2>&1; echo "rc=$?") +ck "batch runs both" "$(printf '%s' "$out" | grep -c '=== vs-main: ')" "2" +ck "batch exits zero" "$(printf '%s' "$out" | grep -c 'rc=0')" "1" + +rm -f "results-vs-main/$EVAL".*.json "results-vs-main/$EVAL2".*.json "results/claude-code-sonnet-5/$EVAL.json" "results/claude-code-sonnet-5/$EVAL2.json" +echo "vs-main.test: $pass passed, $fail failed" +[ "$fail" = 0 ] From 63e30afee209b985b8ad2aaa12756ed9c0999580 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Sat, 25 Jul 2026 11:34:39 +0200 Subject: [PATCH 15/18] fix: slot ports below the macOS ephemeral range; live-proof fixes The 55xxx slot blocks sat inside the macOS ephemeral port range (49152+): observed live, a transient outbound connection held 55422 and the slot-1 db container could not bind. Slot ports now live at 43k21-43k24; the docs-api port derives from the project-id suffix instead of port arithmetic, and the profile's port regex accepts any block. Also from the live proof (isolate -> docs-up -> copy-index on slot 1, primary running throughout): fresh numeric slot allocation was colorized by FORCE_COLOR into NaN ports (String() it), an apostrophe broke the allocator quoting, and setval noise is silenced. Verified: overlay stack boots on the symlinked workdir, free copy lands 1742 pages/$0.00, both stacks coexist, teardown leaves zero containers/volumes. --- workspace/scripts/docs-copy-index.sh | 3 +-- workspace/scripts/docs-isolate.sh | 6 +++++- workspace/scripts/docs-profile.sh | 9 ++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/workspace/scripts/docs-copy-index.sh b/workspace/scripts/docs-copy-index.sh index fd0915ba..9a7532a0 100755 --- a/workspace/scripts/docs-copy-index.sh +++ b/workspace/scripts/docs-copy-index.sh @@ -26,10 +26,9 @@ docker exec "$src" pg_dump -U postgres -d postgres --data-only -t public.page -t docker exec "$dst" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q \ -c 'TRUNCATE public.page, public.page_section RESTART IDENTITY CASCADE' docker exec -i "$dst" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q < "$tmp" -docker exec "$dst" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q -c " +docker exec "$dst" psql -U postgres -d postgres -v ON_ERROR_STOP=1 -q -o /dev/null -c " select setval(pg_get_serial_sequence('public.page','id'), coalesce(max(id),1)) from public.page; select setval(pg_get_serial_sequence('public.page_section','id'), coalesce(max(id),1)) from public.page_section;" - got=$(docker exec "$dst" psql -U postgres -d postgres -tAc 'select count(*) from public.page') [ "$got" = "$pages" ] || { echo "copy incomplete: source $pages pages, destination $got" >&2; exit 1; } echo "copied $got pages -> $dst (\$0.00 — the next docs-index re-embeds only pages your tree changed)" diff --git a/workspace/scripts/docs-isolate.sh b/workspace/scripts/docs-isolate.sh index 556dafdc..fe70247c 100755 --- a/workspace/scripts/docs-isolate.sh +++ b/workspace/scripts/docs-isolate.sh @@ -70,12 +70,16 @@ for f in buckets functions migrations seed.sql; do ln -sfn "../../../submodules/supabase/supabase/$f" "$ov/$f" done +# Slot ports live at 43k21-43k24 — BELOW the macOS ephemeral range (49152+), +# where the primary's 55xxx block flakily collides with transient outbound +# sockets (observed live: an ESTABLISHED connection held 55422 and the db +# container could not bind). SLOT="$slot" node -e ' const fs = require("fs"); const slot = Number(process.env.SLOT); let t = fs.readFileSync(process.argv[1], "utf8"); t = t.replace(/^project_id = ".*"$/m, `project_id = "eval-workspace-content-${slot}"`); -t = t.replace(/^port = 55[0-9]([0-9][0-9])$/gm, (_, tail) => `port = ${55300 + 100 * slot + Number(tail)}`); +t = t.replace(/^port = 55[0-9]([0-9][0-9])$/gm, (_, tail) => `port = ${43000 + 100 * slot + Number(tail)}`); fs.writeFileSync(process.argv[2], t); ' "$src/config.toml" "$ov/config.toml" diff --git a/workspace/scripts/docs-profile.sh b/workspace/scripts/docs-profile.sh index 64779dbd..06ff563e 100644 --- a/workspace/scripts/docs-profile.sh +++ b/workspace/scripts/docs-profile.sh @@ -13,12 +13,15 @@ else fi _cfg="$CONTENT_WORKDIR/supabase/config.toml" CONTENT_PROJECT_ID=$(sed -n 's/^project_id = "\(.*\)"$/\1/p' "$_cfg" 2>/dev/null | head -1) -CONTENT_API_PORT=$(sed -n 's/^port = \(55[0-9][0-9][0-9]\)$/\1/p' "$_cfg" 2>/dev/null | head -1) +CONTENT_API_PORT=$(sed -n 's/^port = \([0-9][0-9]*\)$/\1/p' "$_cfg" 2>/dev/null | head -1) # fall back to the primary defaults when the submodule isn't cloned yet CONTENT_PROJECT_ID="${CONTENT_PROJECT_ID:-eval-workspace-content}" CONTENT_API_PORT="${CONTENT_API_PORT:-55321}" CONTENT_DB_CONTAINER="supabase_db_${CONTENT_PROJECT_ID}" -# slot 0 (primary, 55321) -> 3001; slot k (55321+100k) -> 3001+k -DOCS_API_PORT=$(( 3001 + (CONTENT_API_PORT - 55321) / 100 )) +# docs-api: primary -> 3001; isolated slot k (project id suffix) -> 3001+k +case "$CONTENT_PROJECT_ID" in + eval-workspace-content-[1-9]) DOCS_API_PORT=$(( 3001 + ${CONTENT_PROJECT_ID##*-} )) ;; + *) DOCS_API_PORT=3001 ;; +esac CONTENT_URL="http://127.0.0.1:${DOCS_API_PORT}/docs/api/graphql" unset _cfg From e55d2f30f1ce79c5d03f396115ae8568c91f94a7 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Sat, 25 Jul 2026 12:59:05 +0200 Subject: [PATCH 16/18] =?UTF-8?q?feat:=20vs-main=20--no-compare=20?= =?UTF-8?q?=E2=80=94=20run=20dirty=20trees=20without=20a=20published=20bas?= =?UTF-8?q?eline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auto-detect+sync+receipt machinery was locked behind published-baseline existence, which excluded exactly the evals you write yourself for a docs edit (custom ones aren't in any published export). --no-compare skips baseline resolution entirely: same treatment sync, same provenance receipt, result-only report. Published attempts drive the default run count only in compare mode (a stale published.json from an earlier run must not leak in). vs-main.test: 18/0. --- mise.toml | 2 +- workspace/README.md | 6 ++-- workspace/scripts/vs-main.sh | 47 +++++++++++++++++++++---------- workspace/scripts/vs-main.test.sh | 9 ++++++ 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/mise.toml b/mise.toml index 2d6f2093..b33aab92 100644 --- a/mise.toml +++ b/mise.toml @@ -49,7 +49,7 @@ description = "Zero-cost self-test of the A/B runner (fakes the eval; only touch run = "workspace/scripts/ab.test.sh" [tasks.vs-main] -description = "Screen … against the latest published result on main — treatment only, no stashes, multi-session safe (flags: --experiment, --runs)" +description = "Screen … against the latest published result on main — treatment only, no stashes, multi-session safe (flags: --experiment, --runs, --no-compare)" run = "workspace/scripts/vs-main.sh" [tasks.vs-main-test] diff --git a/workspace/README.md b/workspace/README.md index ed1cc401..e9f94f14 100644 --- a/workspace/README.md +++ b/workspace/README.md @@ -75,7 +75,7 @@ mcp/docs/skills edits) and diff against the latest published result on evals `main` — no baseline arm, no stashes, no git mutation anywhere. ```bash -mise run vs-main […] [--experiment ] [--runs N] +mise run vs-main […] [--experiment ] [--runs N] [--no-compare] ``` Dirty submodule trees are detected and synced automatically (mcp build, docs @@ -83,7 +83,9 @@ re-embed); receipts land in `results-vs-main/*.json` with the published arm's result commit + parent SHA and age. The published arm ran in the scheduled CI world (published mcp package, prod docs index, model state at refresh time), so a flip is a **screen** — confirm causal claims with one paired `mise run -ab`. Free self-test: `mise run vs-main-test`. +ab`. `--no-compare` runs your edited world with the same auto-sync and +receipts but no published row required — the way to run custom evals (not in +any published set) against dirty trees. Free self-test: `mise run vs-main-test`. ## Parallel sessions (worktrees) diff --git a/workspace/scripts/vs-main.sh b/workspace/scripts/vs-main.sh index 8ec42d93..015122c4 100755 --- a/workspace/scripts/vs-main.sh +++ b/workspace/scripts/vs-main.sh @@ -23,16 +23,17 @@ set -euo pipefail cd "$(dirname "$0")/../.." -EXP=claude-code-sonnet-5; RUNS=""; EVALS=() +EXP=claude-code-sonnet-5; RUNS=""; COMPARE=1; EVALS=() while [ $# -gt 0 ]; do case "$1" in --experiment) EXP="${2:?--experiment needs a value}"; shift 2 ;; --runs) RUNS="${2:?--runs needs a value}"; shift 2 ;; + --no-compare) COMPARE=0; shift ;; -*) echo "unknown flag: $1" >&2; exit 2 ;; *) EVALS+=("$1"); shift ;; esac done -[ ${#EVALS[@]} -gt 0 ] || { echo "usage: workspace/scripts/vs-main.sh [...] [--experiment ] [--runs N]" >&2; exit 2; } +[ ${#EVALS[@]} -gt 0 ] || { echo "usage: workspace/scripts/vs-main.sh [...] [--experiment ] [--runs N] [--no-compare]" >&2; exit 2; } for k in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do v="$(security find-generic-password -a "$USER" -s "eval-workspace:$k" -w 2>/dev/null || true)" @@ -42,6 +43,9 @@ done OUT=results-vs-main; mkdir -p "$OUT" # --- resolve published baselines (free; refuses before any spend) --- +# --no-compare: skip entirely — run YOUR world with the same sync/receipts, +# no published row required (custom evals aren't in the published set). +if [ "$COMPARE" = 1 ]; then [ -n "${VSMAIN_NO_FETCH:-}" ] || git fetch -q origin main EXP="$EXP" OUT="$OUT" node - "${EVALS[@]}" <<'EOF' const fs = require("fs"); @@ -84,6 +88,7 @@ for (const e of evals) { } process.exit(missing ? 1 : 0); EOF +fi # --- zero-cost eval validation (same gate as ab.sh) --- if [ -z "${VSMAIN_EVAL_CMD:-}" ]; then @@ -123,14 +128,20 @@ if [ -z "${VSMAIN_SKIP_SYNC:-}" ]; then if [ -e submodules/agent-skills/.git ] && [ -n "$(git -C submodules/agent-skills status --porcelain)" ]; then DIRTY+=(skills) # read live via symlink; no sync step fi - [ ${#DIRTY[@]} -gt 0 ] || echo "note: no local edits detected in submodules/{mcp,supabase/apps/docs/content,agent-skills} — this compares your pinned world against published" + if [ ${#DIRTY[@]} -eq 0 ]; then + [ "$COMPARE" = 1 ] && echo "note: no local edits detected in submodules/{mcp,supabase/apps/docs/content,agent-skills} — this compares your pinned world against published" \ + || echo "note: no local edits detected in submodules/{mcp,supabase/apps/docs/content,agent-skills} — this runs your pinned world as-is" + fi fi # --- run treatment + report, one eval at a time --- FAILED=0 for e in "${EVALS[@]}"; do runs="$RUNS" - [ -n "$runs" ] || runs=$(node -pe 'require(`./${process.env.OUT}/${process.argv[1]}.published.json`).attempts || 1' "$e" 2>/dev/null || echo 1) + # published attempts drive the default only in compare mode (a stale + # published.json from an earlier compare run must not leak in) + [ -n "$runs" ] || { [ "$COMPARE" = 1 ] && runs=$(node -pe 'require(`./${process.env.OUT}/${process.argv[1]}.published.json`).attempts || 1' "$e" 2>/dev/null) || true; } + [ -n "$runs" ] || runs=1 RES="results/$EXP/$e.json" echo "== treatment: $e ($EXP, runs=$runs, edits: ${DIRTY[*]:-none}) ==" if [ -n "${VSMAIN_EVAL_CMD:-}" ]; then @@ -142,22 +153,28 @@ for e in "${EVALS[@]}"; do cp "$RES" "$OUT/$e.treatment.json" env ${RUN_ENV[@]+"${RUN_ENV[@]}"} node workspace/scripts/provenance.mjs --embed "$OUT/$e.treatment.json" - EVAL="$e" EXP="$EXP" OUT="$OUT" DIRTY="${DIRTY[*]:-none}" node -e ' + EVAL="$e" EXP="$EXP" OUT="$OUT" DIRTY="${DIRTY[*]:-none}" COMPARE="$COMPARE" node -e ' const path=require("path"); const f=(p)=>{try{return require(path.resolve(p))}catch{return null}}; -const {EVAL,EXP,OUT,DIRTY}=process.env; -const b=f(`${OUT}/${EVAL}.published.json`), t=f(`${OUT}/${EVAL}.treatment.json`); +const {EVAL,EXP,OUT,DIRTY,COMPARE}=process.env; +const b=COMPARE==="1"?f(`${OUT}/${EVAL}.published.json`):null, t=f(`${OUT}/${EVAL}.treatment.json`); const chk=(r)=>{const c=(r&&r.checks)||[];return `${c.filter(x=>x&&x.passed).length}/${c.length}`}; const row=(l,r,extra)=>`${l.padEnd(10)} passed=${String(r&&r.passed).padEnd(5)} checks=${chk(r).padEnd(6)} docs.calls=${String(((r&&r.docs&&r.docs.calls)||[]).length).padEnd(3)} ${extra||""}`; -const m=(b&&b.vsMainBaseline)||{}; -const age=m.committedAt?Math.round((Date.now()-new Date(m.committedAt))/864e5):"?"; console.log(`\n=== vs-main: ${EVAL} (${EXP}) ===`); -console.log(row("published",b,`main@${(m.commit||"").slice(0,7)} ${String(m.committedAt||"").slice(0,10)} (${age}d old, attempts ${b&&b.attempts})`)); -console.log(row("treatment",t,`your world (edits: ${DIRTY})`)); -const d=((t&&t.passed)?1:0)-((b&&b.passed)?1:0); -console.log(d>0?"-> IMPROVED vs published (FAIL->PASS)":d<0?"-> REGRESSED vs published (PASS->FAIL)":"-> no pass/fail change (compare checks / docs.calls)"); -console.log(`screen only: the published arm ran in the scheduled CI world — confirm causal claims with: mise run ab ${EVAL} `); -console.log(`saved: ${OUT}/${EVAL}.{published,treatment}.json`); +if (b) { + const m=b.vsMainBaseline||{}; + const age=m.committedAt?Math.round((Date.now()-new Date(m.committedAt))/864e5):"?"; + console.log(row("published",b,`main@${(m.commit||"").slice(0,7)} ${String(m.committedAt||"").slice(0,10)} (${age}d old, attempts ${b&&b.attempts})`)); + console.log(row("treatment",t,`your world (edits: ${DIRTY})`)); + const d=((t&&t.passed)?1:0)-((b&&b.passed)?1:0); + console.log(d>0?"-> IMPROVED vs published (FAIL->PASS)":d<0?"-> REGRESSED vs published (PASS->FAIL)":"-> no pass/fail change (compare checks / docs.calls)"); + console.log(`screen only: the published arm ran in the scheduled CI world — confirm causal claims with: mise run ab ${EVAL} `); + console.log(`saved: ${OUT}/${EVAL}.{published,treatment}.json`); +} else { + console.log(row("treatment",t,`your world (edits: ${DIRTY})`)); + console.log(`no comparison (--no-compare): result + provenance receipt only`); + console.log(`saved: ${OUT}/${EVAL}.treatment.json`); +} ' done exit "$FAILED" diff --git a/workspace/scripts/vs-main.test.sh b/workspace/scripts/vs-main.test.sh index b223ec83..3bf07a6c 100755 --- a/workspace/scripts/vs-main.test.sh +++ b/workspace/scripts/vs-main.test.sh @@ -40,5 +40,14 @@ ck "batch runs both" "$(printf '%s' "$out" | grep -c '=== vs-main: ')" "2" ck "batch exits zero" "$(printf '%s' "$out" | grep -c 'rc=0')" "1" rm -f "results-vs-main/$EVAL".*.json "results-vs-main/$EVAL2".*.json "results/claude-code-sonnet-5/$EVAL.json" "results/claude-code-sonnet-5/$EVAL2.json" + +# --- --no-compare: custom eval (no published row anywhere) still runs --- +out=$(bash workspace/scripts/vs-main.sh custom-eval-not-published --no-compare 2>&1; echo "rc=$?") +ck "no-compare runs unpublished eval" "$(printf '%s' "$out" | grep -c '=== vs-main: custom-eval-not-published')" "1" +ck "no-compare has no published row" "$(printf '%s' "$out" | grep -c '^published ')" "0" +ck "no-compare says receipt only" "$(printf '%s' "$out" | grep -c 'no comparison (--no-compare)')" "1" +ck "no-compare exits zero" "$(printf '%s' "$out" | grep -c 'rc=0')" "1" +ck "no-compare treatment receipt exists" "$([ -f results-vs-main/custom-eval-not-published.treatment.json ] && echo 1 || echo 0)" "1" +rm -f results-vs-main/custom-eval-not-published.*.json results/claude-code-sonnet-5/custom-eval-not-published.json echo "vs-main.test: $pass passed, $fail failed" [ "$fail" = 0 ] From f4a146874ef8ec71fada3b992d5b5efe1b1a0d58 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Sat, 25 Jul 2026 13:02:40 +0200 Subject: [PATCH 17/18] =?UTF-8?q?feat:=20mise=20run=20experiments=20?= =?UTF-8?q?=E2=80=94=20list=20experiments=20with=20model/effort=20+=20publ?= =?UTF-8?q?ished-baseline=20marker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mise.toml | 4 ++++ workspace/scripts/experiments.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100755 workspace/scripts/experiments.sh diff --git a/mise.toml b/mise.toml index b33aab92..781c5133 100644 --- a/mise.toml +++ b/mise.toml @@ -60,6 +60,10 @@ run = "workspace/scripts/vs-main.test.sh" description = "Give THIS worktree its own docs stack (project id + ports) for interference-free parallel sessions" run = "workspace/scripts/docs-isolate.sh" +[tasks.experiments] +description = "List available experiments (agent, model, effort) and which have published baselines for vs-main" +run = "workspace/scripts/experiments.sh" + [tasks.status-test] description = "Self-test of status.sh's Next/Ready diagnosis against synthetic workspace states" run = "workspace/scripts/status.test.sh" diff --git a/workspace/scripts/experiments.sh b/workspace/scripts/experiments.sh new file mode 100755 index 00000000..76292944 --- /dev/null +++ b/workspace/scripts/experiments.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# List available experiments (experiments/*.ts): agent, model, reasoning +# effort, and whether evals main has a published row for it (vs-main compare +# mode needs one; anything else works with --no-compare or mise run ab). +set -euo pipefail +cd "$(dirname "$0")/../.." + +published=$(node -e ' +const { execFileSync } = require("child_process"); +const s = new Set(); +for (const f of ["apps/web/src/data/regression-eval-results.json", "apps/web/src/data/eval-results.json"]) { + try { for (const r of JSON.parse(execFileSync("git", ["show", `origin/main:${f}`], { maxBuffer: 1 << 28 }))) s.add(r.experiment); } + catch {} +} +console.log([...s].join(" ")); +' 2>/dev/null || true) + +printf '%-36s %-13s %-22s %-8s %s\n' EXPERIMENT AGENT MODEL EFFORT PUBLISHED +for f in experiments/*.ts; do + name=$(basename "$f" .ts) + agent=$(sed -n 's/.*agent: \([a-zA-Z]*\)Agent(.*/\1/p' "$f" | head -1) + model=$(sed -n "s/.*model: \([a-z]*(\)\{0,1\}'\([^']*\)'.*/\2/p" "$f" | head -1) + case " $published " in + *" $name "*) pub="yes (vs-main)" ;; + *) pub="-" ;; + esac + printf '%-36s %-13s %-22s %-8s %s\n' "$name" "${agent:-?}" "${model:-?}" "${effort:--}" "$pub" +done From 4b2a1de2e300850ee6b9f0b55be1f22dbcb57752 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Sat, 25 Jul 2026 13:03:00 +0200 Subject: [PATCH 18/18] =?UTF-8?q?fix:=20experiments=20lister=20=E2=80=94?= =?UTF-8?q?=20restore=20effort=20column=20(clobbered=20by=20the=20model-re?= =?UTF-8?q?gex=20fix)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- workspace/scripts/experiments.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/workspace/scripts/experiments.sh b/workspace/scripts/experiments.sh index 76292944..1c395d77 100755 --- a/workspace/scripts/experiments.sh +++ b/workspace/scripts/experiments.sh @@ -20,6 +20,7 @@ for f in experiments/*.ts; do name=$(basename "$f" .ts) agent=$(sed -n 's/.*agent: \([a-zA-Z]*\)Agent(.*/\1/p' "$f" | head -1) model=$(sed -n "s/.*model: \([a-z]*(\)\{0,1\}'\([^']*\)'.*/\2/p" "$f" | head -1) + effort=$(sed -n "s/.*reasoningEffort: '\([^']*\)'.*/\1/p" "$f" | head -1) case " $published " in *" $name "*) pub="yes (vs-main)" ;; *) pub="-" ;;