supabase · barryroodt · Jul 24, 2026 · Jul 24, 2026 · Jul 24, 2026 · Jul 24, 2026
diff --git a/.gitignore b/.gitignore
@@ -7,7 +7,8 @@ results/*/
 .sync-tmp/
 
 # eval-source workspace glue (workspace/README.md)
-/supabase/
 /results-ab/
+/results-vs-main/
 /.docs-index-stamp.json
+/workspace/.docs-stack/
 /.publish/
diff --git a/.gitmodules b/.gitmodules
@@ -5,3 +5,8 @@
 	path = submodules/mcp
 	url = git@github.com:supabase/mcp.git
 	ignore = all
+[submodule "submodules/supabase"]
+	path = submodules/supabase
+	url = git@github.com:supabase/supabase.git
+	ignore = all
+	update = none
diff --git a/mise.toml b/mise.toml
@@ -21,7 +21,7 @@ description = "Run evals with the workspace env applied (args pass to `pnpm eval
 run = "workspace/scripts/eval.sh"
 
 [tasks.update]
-description = "Fetch + fast-forward the supabase clone, re-applying patches (-- --check = report only)"
+description = "Update the docs submodule: fetch + rebase local work onto upstream (-- --check = report only)"
 run = "workspace/scripts/update.sh"
 
 [tasks.publish]
@@ -48,6 +48,22 @@ run = "workspace/scripts/ab-demo.sh"
 description = "Zero-cost self-test of the A/B runner (fakes the eval; only touches a clean skill file)"
 run = "workspace/scripts/ab.test.sh"
 
+[tasks.vs-main]
+description = "Screen <eval-id>… against the latest published result on main — treatment only, no stashes, multi-session safe (flags: --experiment, --runs, --no-compare)"
+run = "workspace/scripts/vs-main.sh"
+
+[tasks.vs-main-test]
+description = "Zero-cost self-test of the vs-main screen (fakes the eval run; reads real published baselines)"
+run = "workspace/scripts/vs-main.test.sh"
+
+[tasks.docs-isolate]
+description = "Give THIS worktree its own docs stack (project id + ports) for interference-free parallel sessions"
+run = "workspace/scripts/docs-isolate.sh"
+
+[tasks.experiments]
+description = "List available experiments (agent, model, effort) and which have published baselines for vs-main"
+run = "workspace/scripts/experiments.sh"
+
 [tasks.status-test]
 description = "Self-test of status.sh's Next/Ready diagnosis against synthetic workspace states"
 run = "workspace/scripts/status.test.sh"
@@ -79,7 +95,7 @@ run = "workspace/scripts/mcp-eval.sh"
 # --- Docs loop (optional, heavy — needs Docker + the supabase CLI) ---
 
 [tasks.clone-docs]
-description = "Sparse-clone supabase/supabase (apps/docs + deps) and install. Run once"
+description = "Seed the sparse supabase docs submodule (apps/docs + deps) and install. Run once"
 run = "workspace/scripts/clone-docs.sh"
 
 [tasks.docs-up]

diff --git a/submodules/supabase b/submodules/supabase
diff --git a/workspace/README.md b/workspace/README.md
@@ -31,7 +31,7 @@ put keys in the repo-root `.env` (the fallback `status` will route you to).
 |---|---|---|
 | Skills | `submodules/agent-skills` (a working tree: edit in place) | none |
 | MCP server | `submodules/mcp` | `mise run mcp-build` |
-| Docs | `supabase/` (opt-in sparse clone: `mise run clone-docs`) | `mise run docs-index` (cents) |
+| Docs | `submodules/supabase` (opt-in sparse+partial submodule: `mise run clone-docs`) | `mise run docs-index` (cents) |
 
 - **MCP loop**: `mise run mcp-eval -- <args>` builds the submodule (with the
   enabler patches) and runs evals against it via `SUPABASE_MCP_SERVER_PATH`.
@@ -44,7 +44,10 @@ put keys in the repo-root `.env` (the fallback `status` will route you to).
   (or just use `mise run ab`, which wires this automatically). Content DB
   ports are 55321+ to avoid the eval local-stack range (54321-9). Measuring
   docs impact needs a tools-mode (`interface: mcp`) eval whose answer lives
-  only in the docs.
+  only in the docs. The submodule is pinned by evals (`ignore = all`,
+  `update = none`: recursive inits skip it — the seed script owns the sparse
+  checkout); bump the pin with `mise run update`, then `git add
+  submodules/supabase` deliberately.
 
 ## Head-to-head A/B
 
@@ -55,15 +58,51 @@ loop's scope, then
 mise run ab <eval-id> <edited-path>   # treatment (edit applied) vs baseline (edit reverted)
 ```
 
-The edit is always restored (a failed restore fails the run loudly); per-arm
-provenance receipts land in `results-ab/*.json`. Cost: two model runs. No
-args = readiness probe. First time? `mise run ab-demo` is a guided,
-self-cleaning live proof on the docs loop (spend-gated, asks first).
+The edit is restored on every exit path, including Ctrl-C/TERM (a failed
+restore fails the run loudly). The one uncatchable case is SIGKILL mid-run:
+the edit lands in a marked stash, and the next `ab` invocation detects it and
+prints the recovery command. Per-arm provenance receipts land in
+`results-ab/*.json`. Cost: two model runs. No args = readiness probe. First
+time? `mise run ab-demo` is a guided, self-cleaning live proof on the docs
+loop (spend-gated, asks first). Writing your own discriminator eval? Ask for
+one precise, docs-only fact ("name the exact package for X") — vague
+questions make both arms search for minutes before converging.
+
+## Screening against published results (`vs-main`)
+
+The cheap iteration loop: run eval(s) in YOUR edited world (any mix of
+mcp/docs/skills edits) and diff against the latest published result on evals
+`main` — no baseline arm, no stashes, no git mutation anywhere.
+
+```bash
+mise run vs-main <eval-id> [<eval-id>…] [--experiment <id>] [--runs N] [--no-compare]
+```
+
+Dirty submodule trees are detected and synced automatically (mcp build, docs
+re-embed); receipts land in `results-vs-main/*.json` with the published arm's
+result commit + parent SHA and age. The published arm ran in the scheduled CI
+world (published mcp package, prod docs index, model state at refresh time),
+so a flip is a **screen** — confirm causal claims with one paired `mise run
+ab`. `--no-compare` runs your edited world with the same auto-sync and
+receipts but no published row required — the way to run custom evals (not in
+any published set) against dirty trees. Free self-test: `mise run vs-main-test`.
+
+## Parallel sessions (worktrees)
+
+MCP/skills edits are per-worktree by construction (tools-mode evals only —
+CLI/local-stack evals share host ports, one at a time machine-wide). The docs
+stack is shared by default; give a worktree its own (project id + port block
++ docs-api port, allocated through a locked machine-shared registry):
+
+```bash
+mise run docs-isolate                        # untracked overlay; no clone file touched
+mise run docs-up && workspace/scripts/docs-copy-index.sh   # free seed from a sibling stack
+```
 
 ## Patches & publishing
 
-Local changes to the patched repos (the `supabase/` clone and the
-`submodules/mcp` working tree) are tracked as `.patch` files in
+Local changes to the patched repos (the `submodules/supabase` and
+`submodules/mcp` working trees) are tracked as `.patch` files in
 `workspace/patches/` and applied as marker commits — see
 [patches/README.md](./patches/README.md) for the manifest and the
 publish flow (`mise run publish <repo> <topic>`). A pre-push guard in each
@@ -73,7 +112,7 @@ never gets hooks or marker commits.
 ## Provenance
 
 `mise run status -- --json` prints a receipt: host repo SHA + dirty state,
-submodule pins, supabase clone state, patch fingerprints, and the docs-index
+submodule pins, docs submodule state, patch fingerprints, and the docs-index
 stamp (`.docs-index-stamp.json`, scoped to repo docs content only). `ab.sh`
 embeds a per-arm copy into every A/B result, so a wrong-baseline run is
 immediately obvious.

diff --git a/workspace/manifest.json b/workspace/manifest.json
@@ -6,8 +6,8 @@
       "patches": ["mcp-content-api-url"]
     },
     "supabase": {
-      "dir": "supabase",
-      "remote": "git@github.com:supabase/supabase.git",
+      "dir": "submodules/supabase",
+      "kind": "submodule",
       "patches": [
         "supabase-content-local-ports",
         "supabase-docs-index-fail-closed",

diff --git a/workspace/patches/README.md b/workspace/patches/README.md
@@ -1,6 +1,6 @@
 # Enabler patches
 
-Local changes to the patched repos (the `supabase/` clone and the
+Local changes to the patched repos (the `submodules/supabase` and the
 `submodules/mcp` working tree), materialized by
 `workspace/scripts/apply-patches.sh` as identifiable **local commits** at the
 bottom of each repo's branch:
@@ -28,9 +28,9 @@ derive from the kind).
 |---|---|---|---|---|---|
 | `mcp-content-api-url` | `submodules/mcp` | `transports/stdio.ts` | upstream | 3 stdio integration tests (upstream) | `--content-api-url` flag + `SUPABASE_CONTENT_API_URL` — **merged upstream ([mcp#343](https://github.com/supabase/mcp/pull/343), 2026-07-23), not yet in a release**; the patch retires when a release ships the flag and the pin moves past it |
 | `supabase-content-local-ports` | supabase/supabase | `config.toml` | **LOCAL-ONLY** | n/a | dev ports 55321+ (avoid evals' 54321-9) |
-| `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report |
+| `supabase-docs-index-fail-closed` | supabase/supabase | `generate-embeddings.ts`, `partner-integrations.ts` | upstream | none | fail-closed `purgeOldPages` + token/cost report; partner fetch errors fail loud (a swallowed error looked like zero partners and purged their rows) |
 | `supabase-docs-guide-checksum` | supabase/supabase | `guideModelLoader.ts` | upstream | none | guide checksum + `tryCatch` `onError` fix (docs-index edit detection) |
-| `supabase-docs-lint-warnings-skip` | supabase/supabase | `lint-warnings-guide.ts` | **LOCAL-ONLY** | n/a | `DOCS_EMBED_ALLOW_MISSING_SOURCES` skip |
+| `supabase-docs-lint-warnings-skip` | supabase/supabase |  `base.ts`, `lint-warnings-guide.ts`, `partner-integrations.ts`, `generate-embeddings.ts` | **LOCAL-ONLY** | n/a | `DOCS_EMBED_ALLOW_MISSING_SOURCES` skip; the skipped source registers its path scope and the purge excludes it (skip→purge hole) |
 | `supabase-docs-reference-dup-sources` | supabase/supabase | `sources/reference-doc.ts` + test | upstream | **unit + real-output tests** | dedupe duplicate reference source paths (the 18-page `inserted 2/1` fix) |
 
 ## Changing or regenerating a patch

diff --git a/workspace/patches/supabase-docs-index-fail-closed.patch b/workspace/patches/supabase-docs-index-fail-closed.patch
@@ -94,3 +94,27 @@ index d7519d2b15..4dfe6e6ba6 100644
    await purgeOldPages(supabaseClient, pageTable, refreshVersion)
 
    console.log('Embedding generation complete')
+diff --git a/apps/docs/scripts/search/sources/partner-integrations.ts b/apps/docs/scripts/search/sources/partner-integrations.ts
+index ad9964722e..b268a3c51c 100644
+--- a/apps/docs/scripts/search/sources/partner-integrations.ts
++++ b/apps/docs/scripts/search/sources/partner-integrations.ts
+@@ -21,12 +21,18 @@ function getSupabaseClient() {
+
+ export async function fetchPartners() {
+   const supabase = getSupabaseClient()
+-  const { data: partners } = await supabase
++  const { data: partners, error } = await supabase
+     .from('partners')
+     .select('slug,overview')
+     .eq('approved', true)
+     // We want to show technology integrations, not agencies, in search
+     .neq('type', 'expert')
++  // Fail loud: swallowing the error makes a transient API failure look like
++  // "zero partners", and the post-index purge then deletes every existing
++  // /partners/integrations/* row as stale.
++  if (error) {
++    throw new Error(`Failed to fetch partners for indexing: ${error.message}`)
++  }
+   return partners ?? []
+ }
+
diff --git a/workspace/patches/supabase-docs-lint-warnings-skip.patch b/workspace/patches/supabase-docs-lint-warnings-skip.patch
@@ -1,15 +1,117 @@
+diff --git a/apps/docs/scripts/search/generate-embeddings.ts b/apps/docs/scripts/search/generate-embeddings.ts
+index 4dfe6e6ba6..cf6d80b683 100644
+--- a/apps/docs/scripts/search/generate-embeddings.ts
++++ b/apps/docs/scripts/search/generate-embeddings.ts
+@@ -18,6 +18,7 @@ import {
+   logFailedSections,
+ } from './embeddings/utils.js'
+ import { fetchAllSources } from './sources/index.js'
++import { skippedSourcePathScopes } from './sources/base.js'
+
+ const CONFIG = {
+   // OpenAI settings
+@@ -532,7 +533,7 @@ async function generateEmbeddings() {
+     )
+   }
+
+-  await purgeOldPages(supabaseClient, pageTable, refreshVersion)
++  await purgeOldPages(supabaseClient, pageTable, refreshVersion, skippedSourcePathScopes)
+
+   console.log('Embedding generation complete')
+ }
+@@ -591,13 +592,22 @@ function logFailedPages(pageInfoMap: Map<number, PageInfo>, processingResult: Pr
+ async function purgeOldPages(
+   supabaseClient: SupabaseClient,
+   pageTable: string,
+-  refreshVersion: string
++  refreshVersion: string,
++  protectPathScopes: string[] = []
+ ) {
+   console.log(`Removing old pages and their sections`)
+-  const { error: deletePageError } = await supabaseClient
+-    .from(pageTable)
+-    .delete()
+-    .filter('version', 'neq', refreshVersion)
++  // A source skipped under DOCS_EMBED_ALLOW_MISSING_SOURCES never entered
++  // preparation, so its still-valid rows carry an old version — exclude its
++  // path scope from the purge instead of deleting them. Everything else
++  // (removed/renamed pages) purges as usual, keeping A/B baselines clean.
++  if (protectPathScopes.length > 0) {
++    console.warn(`Purge excluding skipped source path scope(s): ${protectPathScopes.join(', ')}`)
++  }
++  let query = supabaseClient.from(pageTable).delete().filter('version', 'neq', refreshVersion)
++  for (const scope of protectPathScopes) {
++    query = query.not('path', 'like', scope)
++  }
++  const { error: deletePageError } = await query
+   if (deletePageError) throw deletePageError
+ }
+
+diff --git a/apps/docs/scripts/search/sources/base.ts b/apps/docs/scripts/search/sources/base.ts
+index a7835ccd92..c0bb41ead7 100644
+--- a/apps/docs/scripts/search/sources/base.ts
++++ b/apps/docs/scripts/search/sources/base.ts
+@@ -31,3 +31,9 @@ export abstract class BaseSource {
+
+   abstract extractIndexedContent(): string
+ }
++
++// Path scopes (SQL LIKE patterns) of sources skipped under
++// DOCS_EMBED_ALLOW_MISSING_SOURCES. generate-embeddings excludes these from
++// the old-pages purge: a skipped source never enters preparation, so its
++// still-valid rows carry an old version and would otherwise be deleted.
++export const skippedSourcePathScopes: string[] = []
 diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts
-index d28a57a27f..0207f87504 100644
+index d28a57a27f..0a16468494 100644
 --- a/apps/docs/scripts/search/sources/lint-warnings-guide.ts
 +++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts
-@@ -29,6 +29,10 @@ export class LintWarningsGuideLoader extends BaseLoader {
+@@ -3,7 +3,7 @@ import { Octokit } from '@octokit/core'
+ import { retry } from '@octokit/plugin-retry'
+ import crypto, { createHash } from 'node:crypto'
+ import { OCTOKIT_RETRY_OPTIONS } from '../../../lib/octokit.constants.js'
+-import { BaseLoader, BaseSource } from './base.js'
++import { BaseLoader, BaseSource, skippedSourcePathScopes } from './base.js'
+
+ const RetryOctokit = Octokit.plugin(retry)
+
+@@ -29,6 +29,13 @@ export class LintWarningsGuideLoader extends BaseLoader {
 
    async load() {
      if (!appId || !installationId || !privateKey) {
 +      if (process.env.DOCS_EMBED_ALLOW_MISSING_SOURCES) {
 +        console.warn('Skipping lint-warnings guide source: DOCS_GITHUB_APP_* not set (DOCS_EMBED_ALLOW_MISSING_SOURCES)')
++        // Protect this loader's pages (`<path>?queryGroups=lint&lint=<name>`)
++        // from the purge; the `?` keeps the base guide page purgeable.
++        skippedSourcePathScopes.push(`${this.path}?%`)
 +        return []
 +      }
        throw new Error('Missing DOCS_GITHUB_APP_* environment variables')
      }
 
+diff --git a/apps/docs/scripts/search/sources/partner-integrations.ts b/apps/docs/scripts/search/sources/partner-integrations.ts
+index b268a3c51c..2f9e970cb1 100644
+--- a/apps/docs/scripts/search/sources/partner-integrations.ts
++++ b/apps/docs/scripts/search/sources/partner-integrations.ts
+@@ -1,7 +1,7 @@
+ import { type SupabaseClient, createClient } from '@supabase/supabase-js'
+ import { upperFirst } from 'lodash-es'
+ import { processMdx } from '../../helpers.mdx.js'
+-import { BaseLoader, BaseSource } from './base.js'
++import { BaseLoader, BaseSource, skippedSourcePathScopes } from './base.js'
+
+ type PartnerData = {
+   slug: string // The partner slug corresponding to the last part of the URL
+@@ -31,6 +31,13 @@ export async function fetchPartners() {
+   // "zero partners", and the post-index purge then deletes every existing
+   // /partners/integrations/* row as stale.
+   if (error) {
++    if (process.env.DOCS_EMBED_ALLOW_MISSING_SOURCES) {
++      console.warn(
++        `Skipping partner integrations source: ${error.message} (DOCS_EMBED_ALLOW_MISSING_SOURCES)`
++      )
++      skippedSourcePathScopes.push('/partners/integrations/%')
++      return []
++    }
+     throw new Error(`Failed to fetch partners for indexing: ${error.message}`)
+   }
+   return partners ?? []
diff --git a/workspace/scripts/ab-demo.sh b/workspace/scripts/ab-demo.sh
@@ -16,7 +16,7 @@
 set -euo pipefail
 cd "$(dirname "$0")/../.."
 
-GUIDE=supabase/apps/docs/content/guides/auth/choosing-a-server-package.mdx
+GUIDE=submodules/supabase/apps/docs/content/guides/auth/choosing-a-server-package.mdx
 EVAL_ID=investigate-workspace-canary-nimbus-package
 EVAL_DST=evals/$EVAL_ID
 
@@ -29,14 +29,15 @@ for k in ANTHROPIC_API_KEY OPENAI_API_KEY; do
 done
 
 # --- hard preflight (same gates the A/B itself needs) ---
-[ -e supabase/.git ] || fail "supabase not cloned"
+[ -e submodules/supabase/.git ] || fail "supabase not cloned"
 [ -n "${ANTHROPIC_API_KEY:-}" ] || fail "ANTHROPIC_API_KEY missing"
 [ -n "${OPENAI_API_KEY:-}" ] || fail "OPENAI_API_KEY missing"
-docker exec supabase_db_eval-workspace-content true 2>/dev/null || fail "content DB not running"
-pages=$(docker exec supabase_db_eval-workspace-content psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0)
+source workspace/scripts/docs-profile.sh
+docker exec "$CONTENT_DB_CONTAINER" true 2>/dev/null || fail "content DB not running"
+pages=$(docker exec "$CONTENT_DB_CONTAINER" psql -U postgres -d postgres -tAc 'select count(*) from public.page' 2>/dev/null || echo 0)
 [ "${pages:-0}" -gt 0 ] 2>/dev/null || fail "docs index not seeded"
-curl -sf -o /dev/null http://127.0.0.1:3001/docs/api/graphql || fail "docs-api not serving on :3001"
-git -C supabase diff --quiet -- "${GUIDE#supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}"
+curl -sf -o /dev/null "$CONTENT_URL" || fail "docs-api not serving on :$DOCS_API_PORT"
+git -C submodules/supabase diff --quiet -- "${GUIDE#submodules/supabase/}" || fail "demo guide has local edits (demo needs a clean file): ${GUIDE}"
 [ ! -e "$EVAL_DST" ] || fail "$EVAL_DST already exists — remove it first"
 
 echo "This runs a live A/B: 2 claude-sonnet-5 runs + a few embedding cents, ~3-5 min."
@@ -46,7 +47,8 @@ read -r -p "Type 'demo' to proceed: " confirmation
 # --- plant the canary (append-only; reverted by the cleanup trap) ---
 cleanup() {
   echo "== demo cleanup: reverting canary, de-embedding, removing throwaway eval =="
-  git -C supabase checkout -- "${GUIDE#supabase/}" 2>/dev/null || true
+  git -C submodules/supabase checkout -- "${GUIDE#submodules/supabase/}" \
+    || echo "WARNING: canary revert FAILED — the de-embed below re-indexes the canary; check ${GUIDE}" >&2
   rm -rf "$EVAL_DST"
   workspace/scripts/docs-index.sh >/dev/null 2>&1 \
     || echo "WARNING: cleanup re-embed failed — run \`mise run docs-index\` to de-embed the canary" >&2