From f4a367066d333695ef1a171ecda09521a1dd6845 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 23 Jul 2026 20:31:00 +0100 Subject: [PATCH 1/9] feat: add Vercel Sandbox eval runner (AI-912 spike) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dispatch (experiment x eval) pairs to Vercel Sandbox microVMs instead of GitHub Actions matrix jobs. Each pair gets its own Firecracker VM that plays the runner's role: clone at the pushed commit, dnf-install Docker and start dockerd, pnpm install, run the same `pnpm eval` command line as eval-refresh.yml, then pull results/ back (tar + downloadFile) and run the publish-results export step scoped to the dispatched pairs. packages/sandbox is untouched: the agent container and the Supabase CLI's sibling containers run against the VM's own dockerd exactly as they do against a GitHub runner's daemon. Vercel credentials go to Sandbox.create explicitly — the SDK reads no VERCEL_* env vars, and CI has no cached dev credentials. Validated E2E: two pairs in parallel; build-cli-001-bootstrap-app (CLI/ local-stack) passed 7/7 inside the VM; investigate-db-001 scored identically to local controls on haiku (FAIL 0/3) and sonnet (PASS 3/3). Findings recorded in packages/vercel-runner/README.md. Co-Authored-By: Claude Fable 5 --- .env.example | 7 + .gitignore | 2 + package.json | 1 + packages/vercel-runner/README.md | 102 +++++++ packages/vercel-runner/package.json | 18 ++ packages/vercel-runner/src/discover.ts | 132 +++++++++ packages/vercel-runner/src/run.ts | 224 +++++++++++++++ packages/vercel-runner/src/sandbox-job.ts | 316 ++++++++++++++++++++++ packages/vercel-runner/tsconfig.json | 4 + pnpm-lock.yaml | 154 +++++++++++ 10 files changed, 960 insertions(+) create mode 100644 packages/vercel-runner/README.md create mode 100644 packages/vercel-runner/package.json create mode 100644 packages/vercel-runner/src/discover.ts create mode 100644 packages/vercel-runner/src/run.ts create mode 100644 packages/vercel-runner/src/sandbox-job.ts create mode 100644 packages/vercel-runner/tsconfig.json diff --git a/.env.example b/.env.example index 19a0696f..a30eeb4e 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,10 @@ # AI SDK Core direct-provider credentials. ANTHROPIC_API_KEY= OPENAI_API_KEY= + +# Vercel Sandbox eval runner (pnpm eval:vercel) — see packages/vercel-runner. +VERCEL_TOKEN= +VERCEL_TEAM_ID= +VERCEL_PROJECT_ID= +# Token able to read this repo; the sandbox clones it over HTTPS (`gh auth token` works). +GITHUB_TOKEN= diff --git a/.gitignore b/.gitignore index 932a6aea..76f7a5b8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ dist/ results/*/ .sync-tmp/ +.vercel +.env* diff --git a/package.json b/package.json index cf70715a..8c24c2a5 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry", "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke", "eval:force": "pnpm --filter @supabase-evals/framework eval:force", + "eval:vercel": "pnpm --filter @supabase-evals/vercel-runner run-evals", "test:framework": "pnpm --filter @supabase-evals/framework test:framework", "export-results": "pnpm --filter @supabase-evals/framework export-results", "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", diff --git a/packages/vercel-runner/README.md b/packages/vercel-runner/README.md new file mode 100644 index 00000000..f823d99a --- /dev/null +++ b/packages/vercel-runner/README.md @@ -0,0 +1,102 @@ +# @supabase-evals/vercel-runner + +Spike for [AI-912](https://linear.app/supabase/issue/AI-912/spike-vercel-sandbox-runner-for-evals): +dispatch eval runs to [Vercel Sandbox](https://vercel.com/docs/sandbox) +microVMs instead of GitHub Actions matrix jobs. + +`eval-refresh-vercel.yml` runs this dispatcher from a single GitHub Actions +job — a manually-dispatched twin of `eval-refresh.yml` (same inputs and +publish steps) that can take over the schedule/PR triggers once promoted. The +sections below explain what the Actions matrix does and how each piece maps +onto Sandbox. + +## How the GitHub Actions matrix maps onto Vercel Sandbox + +Today (`.github/workflows/eval-refresh.yml`): + +1. A `prepare` job discovers `(experiment × eval)` pairs from eval `PROMPT.md` + frontmatter and `pnpm eval -- list`. +2. A `run-evals` matrix job runs each pair on its own `ubuntu-latest` runner: + checkout (with submodules), pnpm install, `pnpm eval -- --experiment … + --eval … --runs … --timeout-sec …`, upload `results//` as an + artifact. +3. A `publish-results` job merges the artifacts and exports/commits JSON. + +Docker plays two roles on each runner (see `packages/sandbox`): + +- The harness starts the **agent's sandbox container** (Docker-out-of-Docker: + the host socket is mounted into it). +- For `interface: cli` evals, the Supabase CLI *inside* that container spawns + the local stack (postgres, gotrue, kong, …) as **sibling containers** on the + runner's daemon, with host networking so `127.0.0.1` ports line up. + +This runner replaces step 2's machine, one Firecracker microVM per pair: + +| GitHub Actions | Vercel Sandbox | +| --------------------------------- | -------------------------------------------------- | +| `prepare` matrix discovery | `src/discover.ts` (same suite rules) | +| `ubuntu-latest` runner | `Sandbox.create()` VM (Amazon Linux 2023) | +| `actions/checkout` + submodules | `source: { type: "git", revision }` + submodule init | +| runner's built-in Docker daemon | `dnf install docker` + detached `dockerd` | +| `strategy.matrix` parallelism | N sandboxes in flight (`--concurrency`) | +| artifact upload/download | `tar` + `sandbox.readFileToBuffer()` → `results/` | + +`packages/sandbox` is untouched: the agent container and the Supabase sibling +containers run against the VM's own dockerd exactly as they do against a +runner's daemon. + +## Usage + +```bash +pnpm eval:vercel -- \ + --experiment claude-haiku-4.5 \ + --eval investigate-db-001-table-row-counts,build-cli-001-bootstrap-app \ + --runs 1 --timeout-sec 720 +``` + +Omit `--experiment`/`--eval` to fan out over the same matrix the scheduled +workflow would (`--suite`, `--experiment-suite` filter it). `--dry` prints the +plan without dispatching. The sandbox runs the **pushed commit** (`--revision` +overrides; defaults to `HEAD`, which must be on a remote branch). + +Required in `.env`: + +| Variable | Purpose | +| ----------------------------------------------- | ---------------------------------------------- | +| `VERCEL_TOKEN` / `VERCEL_TEAM_ID` / `VERCEL_PROJECT_ID` | Sandbox SDK auth (any project works) | +| `GITHUB_TOKEN` | Cloning this internal repo (`gh auth token`) | +| `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` | Forwarded into the sandbox's `.env` for agents | + +## Spike findings + +Validated end-to-end with two pairs in parallel on `claude-haiku-4.5` +(`investigate-db-001-table-row-counts`, tools mode, and +`build-cli-001-bootstrap-app`, CLI/local-stack mode — the latter passed 7/7 +checks inside the VM): + +- **Docker works in the VM**: `dnf install docker`, detached `dockerd`; the + whole DoD + sibling-container topology of `packages/sandbox` runs unchanged. +- **Results are faithful**: the tools-mode eval produced the same score as a + local control run of the same commit. +- **Long commands must not lean on one log stream**: a multi-minute + `runCommand` held on a single streaming connection dies with "Stream ended + before command finished" while the command keeps running. The eval step + therefore runs detached with output to a file, polled with short commands. +- **Publishing matches CI**: after an all-green run the runner executes the + same `export-results` commands as the `publish-results` job, scoped to the + dispatched pairs (a dev machine's `results/` tree carries older runs, which + an unscoped `--merge` would resurface). Committing/PR-ing the JSON stays + with the caller, as in CI. +- **Limits** (Pro plan): 8 vCPUs / 16 GB / 32 GB disk per sandbox, 24 h max + runtime, 2,000 concurrent sandboxes, 200 vCPUs/min creation rate — far above + what the current matrix needs. +- **Cost** is metered on *active* CPU (~$0.128/h) + provisioned memory + (~$0.021/GB-h); an eval pair (mostly waiting on the model) costs cents. +- **Cold start** is the main overhead: dnf + dockerd + pnpm install + pulling + the Supabase images adds ~4–6 minutes per VM vs. a GitHub runner's warm + image cache. A [sandbox snapshot](https://vercel.com/docs/sandbox/concepts/snapshots) + with docker + node_modules + Supabase images pre-baked would cut this to + seconds and is the obvious next step. +- **Not built here** (out of spike scope): the durable-queue dispatch the + issue floats, and snapshot caching. Committing/PR-ing the exported JSON + stays in the workflow's existing steps, which are unchanged. diff --git a/packages/vercel-runner/package.json b/packages/vercel-runner/package.json new file mode 100644 index 00000000..5d825efc --- /dev/null +++ b/packages/vercel-runner/package.json @@ -0,0 +1,18 @@ +{ + "name": "@supabase-evals/vercel-runner", + "private": true, + "version": "0.0.1", + "type": "module", + "scripts": { + "run-evals": "node --env-file-if-exists=../../.env --import tsx/esm src/run.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@vercel/sandbox": "^2.8.0" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsx": "^4.19.0", + "typescript": "catalog:" + } +} diff --git a/packages/vercel-runner/src/discover.ts b/packages/vercel-runner/src/discover.ts new file mode 100644 index 00000000..1e11cfe6 --- /dev/null +++ b/packages/vercel-runner/src/discover.ts @@ -0,0 +1,132 @@ +/** + * Eval/experiment pair discovery — a TypeScript port of the `prepare` job in + * .github/workflows/eval-refresh.yml, so the Vercel Sandbox dispatcher fans + * out over exactly the same matrix the GitHub Actions workflow would. + */ + +import { execFile } from "node:child_process"; +import { readdirSync, readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export interface EvalPair { + evalId: string; + experiment: string; + experimentSuite: string; + evalSuite: string; +} + +export interface DiscoverOptions { + /** Repo root (contains evals/ and experiments/). */ + root: string; + /** Explicit eval ids; when set, suite filtering of evals is skipped. */ + evalIds?: string[]; + /** Eval suites to include when no explicit ids are given. */ + suites: string[]; + /** Experiment suites to include. */ + experimentSuites: string[]; + /** Explicit experiment names; when set, per-suite discovery is skipped. */ + experiments?: string[]; +} + +/** + * Map an eval suite to the experiment suites it runs under — the same + * hardcoded pairing as the workflow (benchmark evals also run the no-skills + * ablation; regression evals run regression experiments only). + */ +const EXPERIMENT_SUITES_BY_EVAL_SUITE: Record = { + benchmark: ["benchmark", "no-skills"], + regression: ["regression"], +}; + +/** Read the `suite:` frontmatter value of an eval's PROMPT.md. */ +function readEvalSuite(root: string, evalId: string): string | undefined { + const promptPath = join(root, "evals", evalId, "PROMPT.md"); + if (!existsSync(promptPath)) return undefined; + const match = readFileSync(promptPath, "utf8").match(/^suite:\s*(\S+)/m); + return match?.[1]; +} + +/** `pnpm --silent eval -- list --experiment-suite ` at the repo root. */ +async function listExperiments(root: string, suite: string): Promise { + const { stdout } = await execFileAsync( + "pnpm", + ["--silent", "eval", "--", "list", "--experiment-suite", suite], + { cwd: root }, + ); + return JSON.parse(stdout) as string[]; +} + +export async function discoverPairs( + options: DiscoverOptions, +): Promise { + const { root } = options; + + let evalIds: string[]; + if (options.evalIds && options.evalIds.length > 0) { + const missing = options.evalIds.filter( + (id) => !existsSync(join(root, "evals", id)), + ); + if (missing.length > 0) { + throw new Error(`no eval directory for: ${missing.join(", ")}`); + } + evalIds = options.evalIds; + } else { + evalIds = readdirSync(join(root, "evals")).filter((id) => { + const suite = readEvalSuite(root, id); + return suite !== undefined && options.suites.includes(suite); + }); + } + + // Fully explicit selection (both --eval and --experiment): pair directly. + // The suite mapping below exists to mirror the scheduled/dispatch matrix; it + // would silently drop evals outside the benchmark/regression suites, which + // is wrong when the caller has already named exactly what to run. + if ( + options.evalIds && + options.evalIds.length > 0 && + options.experiments && + options.experiments.length > 0 + ) { + return evalIds.flatMap((evalId) => + options.experiments!.map((experiment) => ({ + evalId, + experiment, + experimentSuite: "", + evalSuite: readEvalSuite(root, evalId) ?? "", + })), + ); + } + + const experimentsBySuite = new Map(); + const pairs: EvalPair[] = []; + for (const evalId of evalIds) { + const evalSuite = readEvalSuite(root, evalId); + if (!evalSuite) continue; + const candidateSuites = EXPERIMENT_SUITES_BY_EVAL_SUITE[evalSuite] ?? []; + + for (const experimentSuite of candidateSuites) { + if (!options.experimentSuites.includes(experimentSuite)) continue; + + let experiments: string[]; + if (options.experiments && options.experiments.length > 0) { + experiments = options.experiments; + } else { + if (!experimentsBySuite.has(experimentSuite)) { + experimentsBySuite.set( + experimentSuite, + await listExperiments(root, experimentSuite), + ); + } + experiments = experimentsBySuite.get(experimentSuite)!; + } + + for (const experiment of experiments) { + pairs.push({ evalId, experiment, experimentSuite, evalSuite }); + } + } + } + return pairs; +} diff --git a/packages/vercel-runner/src/run.ts b/packages/vercel-runner/src/run.ts new file mode 100644 index 00000000..15897576 --- /dev/null +++ b/packages/vercel-runner/src/run.ts @@ -0,0 +1,224 @@ +#!/usr/bin/env tsx +/** + * Dispatch eval runs to Vercel Sandbox microVMs — a drop-in stand-in for the + * GitHub Actions matrix in eval-refresh.yml (AI-912 spike). + * + * pnpm eval:vercel -- --experiment claude-haiku-4.5 \ + * --eval investigate-db-001-table-row-counts,build-cli-001-bootstrap-app \ + * --runs 1 + * + * Requires in .env (or the environment): + * VERCEL_TOKEN, VERCEL_TEAM_ID, VERCEL_PROJECT_ID — sandbox auth + * GITHUB_TOKEN — repo checkout (internal repo) + * ANTHROPIC_API_KEY / OPENAI_API_KEY — forwarded to the agents + */ + +import { execFileSync, spawnSync } from "node:child_process"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { discoverPairs, type EvalPair } from "./discover.js"; +import { runPairInSandbox, type SandboxJobResult } from "./sandbox-job.js"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); + +/** Setup (docker + pnpm install) and scoring headroom around the eval runs. */ +const SANDBOX_SETUP_HEADROOM_MIN = 20; + +const rawArgs = process.argv.slice(2).filter((arg) => arg !== "--"); + +function readFlag(name: string): string | undefined { + const prefix = `--${name}=`; + const inline = rawArgs.find((arg) => arg.startsWith(prefix)); + if (inline) return inline.slice(prefix.length); + const index = rawArgs.indexOf(`--${name}`); + if (index !== -1) { + const value = rawArgs[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`--${name} requires a value`); + } + return value; + } + return undefined; +} + +function readList(name: string): string[] { + return (readFlag(name) ?? "") + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function git(...args: string[]): string { + return execFileSync("git", args, { cwd: ROOT, encoding: "utf8" }).trim(); +} + +function requireEnv(name: string, hint: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is not set — ${hint}`); + return value; +} + +async function main() { + const dry = rawArgs.includes("--dry"); + const runs = Number(readFlag("runs") ?? 2); + const timeoutSec = Number(readFlag("timeout-sec") ?? 720); + const vcpus = Number(readFlag("vcpus") ?? 4); + const concurrency = Number(readFlag("concurrency") ?? 4); + const suites = readList("suite"); + const experimentSuites = readList("experiment-suite"); + + const pairs = await discoverPairs({ + root: ROOT, + evalIds: readList("eval"), + experiments: readList("experiment"), + suites: suites.length > 0 ? suites : ["benchmark"], + experimentSuites: + experimentSuites.length > 0 ? experimentSuites : ["benchmark", "no-skills"], + }); + if (pairs.length === 0) throw new Error("no experiment and eval pairs matched"); + + const revision = readFlag("revision") ?? git("rev-parse", "HEAD"); + const sandboxTimeoutMs = + Number( + readFlag("sandbox-timeout-min") ?? + Math.ceil((runs * timeoutSec) / 60) + SANDBOX_SETUP_HEADROOM_MIN, + ) * + 60_000; + + console.log( + `${pairs.length} pair(s), runs=${runs}, timeout=${timeoutSec}s, ` + + `vcpus=${vcpus}, concurrency=${concurrency}, sandbox timeout=${sandboxTimeoutMs / 60000}m, rev=${revision.slice(0, 8)}`, + ); + for (const pair of pairs) { + console.log(`PLAN ${pair.experiment} x ${pair.evalId}`); + } + if (dry) return; + + // The SDK reads VERCEL_TOKEN/VERCEL_TEAM_ID/VERCEL_PROJECT_ID (or an OIDC + // token) from the environment; fail fast with a useful message instead. + if (!process.env.VERCEL_OIDC_TOKEN) { + requireEnv("VERCEL_TOKEN", "create one at https://vercel.com/account/settings/tokens"); + requireEnv("VERCEL_TEAM_ID", "team settings → General → Team ID"); + requireEnv("VERCEL_PROJECT_ID", "project settings → General → Project ID"); + } + const githubToken = requireEnv( + "GITHUB_TOKEN", + "the sandbox clones this internal repo over HTTPS (`gh auth token` works)", + ); + + // The commit must be on the remote for the sandbox to fetch it. + if (!readFlag("revision")) { + const onRemote = git("branch", "-r", "--contains", revision); + if (!onRemote) { + throw new Error( + `HEAD (${revision.slice(0, 8)}) is not on any remote branch — push first, or pass --revision`, + ); + } + if (git("status", "--porcelain")) { + console.warn("warning: working tree is dirty; the sandbox runs the pushed commit, not local changes"); + } + } + + const agentEnv: Record = {}; + for (const key of ["ANTHROPIC_API_KEY", "OPENAI_API_KEY"]) { + if (process.env[key]) agentEnv[key] = process.env[key]!; + } + + const repoUrl = git("remote", "get-url", "origin").replace( + /^git@github\.com:/, + "https://github.com/", + ); + + // Same fan-out/parallelism model as the Actions matrix: every pair gets its + // own isolated machine; only the number in flight at once is capped. + const queue: EvalPair[] = [...pairs]; + const results: SandboxJobResult[] = []; + const workers = Array.from( + { length: Math.min(concurrency, pairs.length) }, + async () => { + let pair: EvalPair | undefined; + while ((pair = queue.shift()) !== undefined) { + results.push( + await runPairInSandbox({ + pair, + repoUrl, + revision, + githubToken, + runs, + timeoutSec, + vcpus, + sandboxTimeoutMs, + agentEnv, + resultsDir: join(ROOT, "results"), + }), + ); + } + }, + ); + await Promise.all(workers); + + console.log("\n=== summary ==="); + for (const result of results) { + const minutes = (result.durationMs / 60000).toFixed(1); + console.log( + `${result.ok ? "✅" : "💥"} ${result.pair.experiment} x ${result.pair.evalId} ` + + `(${minutes}m${result.ok ? "" : `, ${result.error}`})`, + ); + } + const failed = results.filter((result) => !result.ok); + if (failed.length > 0) { + throw new Error(`${failed.length} sandbox job(s) failed`); + } + + exportResults(pairs, rawArgs.includes("--merge")); +} + +/** + * The publish step of eval-refresh.yml's `publish-results` job: export the + * collected results into the web app's data files, per eval suite present in + * the matrix. Like CI, it runs only when every job succeeded, and evals + * outside the benchmark/regression suites are not published. Committing or + * PR-ing the JSON stays with the caller. + */ +const EXPORT_OUTPUT_BY_SUITE: Record = { + benchmark: "apps/web/src/data/eval-results.json", + regression: "apps/web/src/data/regression-eval-results.json", +}; + +function exportResults(pairs: EvalPair[], merge: boolean): void { + for (const [suite, output] of Object.entries(EXPORT_OUTPUT_BY_SUITE)) { + const suitePairs = pairs.filter((pair) => pair.evalSuite === suite); + if (suitePairs.length === 0) continue; + console.log(`\nexporting ${suite} results → ${output}`); + // Unlike CI — whose workspace only ever contains this run's downloaded + // artifacts — a dev machine's results/ tree carries older local runs, so + // scope the export to exactly what this dispatch produced. + const experiments = [...new Set(suitePairs.map((pair) => pair.experiment))]; + const evalIds = [...new Set(suitePairs.map((pair) => pair.evalId))]; + const result = spawnSync( + "pnpm", + [ + "--filter", + "@supabase-evals/framework", + "export-results", + "--", + "--suite", + suite, + ...experiments.flatMap((name) => ["--experiment", name]), + ...evalIds.flatMap((id) => ["--eval", id]), + "--output", + output, + ...(merge ? ["--merge"] : []), + ], + { cwd: ROOT, stdio: "inherit" }, + ); + if (result.status !== 0) { + throw new Error(`export-results failed for suite ${suite}`); + } + } +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); +}); diff --git a/packages/vercel-runner/src/sandbox-job.ts b/packages/vercel-runner/src/sandbox-job.ts new file mode 100644 index 00000000..66798e6c --- /dev/null +++ b/packages/vercel-runner/src/sandbox-job.ts @@ -0,0 +1,316 @@ +/** + * One (experiment × eval) pair inside one Vercel Sandbox microVM — the + * Firecracker equivalent of a `run-evals` matrix job in eval-refresh.yml. + * + * The VM plays the role the GitHub Actions runner plays today: it checks out + * the repo, installs pnpm dependencies, and runs `pnpm eval`. Docker keeps the + * exact same shape it has on a runner — the harness starts the agent's sandbox + * container against the VM's own dockerd, and for local-stack evals the + * Supabase CLI inside that container spawns the stack as sibling containers on + * that same daemon. Nothing in packages/sandbox changes; only the machine the + * daemon runs on does. + */ + +import { mkdirSync, rmSync, mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Writable } from "node:stream"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { Sandbox } from "@vercel/sandbox"; +import type { EvalPair } from "./discover.js"; + +const execFileAsync = promisify(execFile); + +/** Where the git source is checked out inside the sandbox. */ +const SANDBOX_CWD = "/vercel/sandbox"; + +/** Ready-poll budget for dockerd inside the VM. */ +const DOCKERD_READY_TIMEOUT_SEC = 60; + +export interface SandboxJobOptions { + pair: EvalPair; + /** HTTPS clone URL of this repo. */ + repoUrl: string; + /** Commit SHA to run — must be reachable on the remote. */ + revision: string; + /** Token able to read the repo (it's internal) — goes into the git source. */ + githubToken: string; + runs: number; + timeoutSec: number; + vcpus: number; + /** Session timeout for the whole VM. */ + sandboxTimeoutMs: number; + /** Model-provider keys written to the sandbox's .env for `pnpm eval`. */ + agentEnv: Record; + /** Local directory the sandbox's results/ tree is extracted into. */ + resultsDir: string; +} + +/** + * Explicit Vercel API credentials, when the environment carries them. The SDK + * does NOT read VERCEL_TOKEN/VERCEL_TEAM_ID/VERCEL_PROJECT_ID on its own — its + * only fallbacks are $VERCEL_OIDC_TOKEN and interactively cached dev + * credentials (`vercel link`), and CI has neither. + */ +function credentialsFromEnv(): + | { token: string; teamId: string; projectId: string } + | undefined { + const { VERCEL_TOKEN, VERCEL_TEAM_ID, VERCEL_PROJECT_ID } = process.env; + if (!VERCEL_TOKEN || !VERCEL_TEAM_ID || !VERCEL_PROJECT_ID) return undefined; + return { + token: VERCEL_TOKEN, + teamId: VERCEL_TEAM_ID, + projectId: VERCEL_PROJECT_ID, + }; +} + +export interface SandboxJobResult { + pair: EvalPair; + ok: boolean; + /** Failure summary when ok is false. */ + error?: string; + durationMs: number; + sandboxId?: string; +} + +/** Prefix every output line so interleaved parallel jobs stay readable. */ +function prefixedWriter(prefix: string, target: NodeJS.WriteStream): Writable { + let pending = ""; + return new Writable({ + write(chunk, _encoding, callback) { + pending += chunk.toString(); + const lines = pending.split("\n"); + pending = lines.pop() ?? ""; + for (const line of lines) target.write(`${prefix} ${line}\n`); + callback(); + }, + final(callback) { + if (pending) target.write(`${prefix} ${pending}\n`); + callback(); + }, + }); +} + +export async function runPairInSandbox( + options: SandboxJobOptions, +): Promise { + const { pair } = options; + const label = `[${pair.experiment} × ${pair.evalId}]`; + const stdout = prefixedWriter(label, process.stdout); + const stderr = prefixedWriter(label, process.stderr); + const log = (message: string) => stdout.write(`${message}\n`); + const start = Date.now(); + + let sandbox: Sandbox | undefined; + try { + log(`creating sandbox (vcpus=${options.vcpus}, timeout=${Math.round(options.sandboxTimeoutMs / 60000)}m, rev=${options.revision.slice(0, 8)})`); + sandbox = await Sandbox.create({ + ...credentialsFromEnv(), + runtime: "node22", + resources: { vcpus: options.vcpus }, + timeout: options.sandboxTimeoutMs, + source: { + type: "git", + url: options.repoUrl, + revision: options.revision, + // Vercel clones with these as basic-auth; x-access-token is GitHub's + // conventional username for token auth. + username: "x-access-token", + password: options.githubToken, + }, + }); + log(`sandbox ${sandbox.name} created`); + + const run = async ( + step: string, + cmd: string, + opts: { sudo?: boolean; env?: Record } = {}, + ) => { + log(`--- ${step}`); + const result = await sandbox!.runCommand({ + cmd: "bash", + args: ["-c", `set -euo pipefail\n${cmd}`], + cwd: SANDBOX_CWD, + sudo: opts.sudo, + env: opts.env, + stdout, + stderr, + }); + if (result.exitCode !== 0) { + throw new Error(`step "${step}" exited with code ${result.exitCode}`); + } + }; + + // -- Bootstrap: what actions/checkout + setup-node + the runner image give + // us for free on GitHub Actions. + await run("install docker", "dnf install -y -q docker", { sudo: true }); + await sandbox.runCommand({ cmd: "dockerd", sudo: true, detached: true }); + await run( + "wait for dockerd", + `for i in $(seq ${DOCKERD_READY_TIMEOUT_SEC}); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1`, + { sudo: true }, + ); + // The harness shells out to `docker` as the unprivileged user; inside a + // single-tenant ephemeral VM, opening the socket is the simple safe way. + await run("open docker socket", "chmod 666 /var/run/docker.sock", { + sudo: true, + }); + + // The agent-skills submodule is public; rewrite its SSH URL to HTTPS since + // the VM has no GitHub SSH identity. The rewrite must live in --global + // config: the child `git clone` a submodule update spawns doesn't read the + // superproject's repo-local config. + await run( + "checkout submodules", + `git config --global url."https://github.com/".insteadOf "git@github.com:"\ngit submodule update --init --recursive`, + ); + + await run("install pnpm", "npm install -g pnpm@10.24.0", { sudo: true }); + await run("pnpm install", "pnpm install --frozen-lockfile"); + + // `pnpm eval` reads the repo-root .env (node --env-file). + await sandbox.writeFiles([ + { + path: ".env", + content: Buffer.from( + Object.entries(options.agentEnv) + .map(([key, value]) => `${key}=${value}`) + .join("\n") + "\n", + ), + }, + ]); + + // Long-running step, decoupled from any live log stream: launch detached + // with output going to a file in the VM, then poll with short commands + // (status sentinel + incremental tail). A multi-minute `pnpm eval` held on + // one streaming connection dies with "Stream ended before command + // finished" when that connection drops, even though the command and the + // VM are fine. + const runPolled = async (step: string, cmd: string) => { + log(`--- ${step}`); + const logPath = "/tmp/step.log"; + const exitPath = "/tmp/step.exit"; + await sandbox!.runCommand({ + cmd: "bash", + args: [ + "-c", + `rm -f ${logPath} ${exitPath}; (set -euo pipefail\n${cmd}\n) >${logPath} 2>&1; echo $? >${exitPath}`, + ], + cwd: SANDBOX_CWD, + detached: true, + }); + + let offset = 0; + let pollFailures = 0; + const emitNewOutput = async (limit?: number) => { + const chunk = await sandbox!.runCommand({ + cmd: "bash", + args: [ + "-c", + `tail -c +${offset + 1} ${logPath}${limit ? ` | head -c ${limit}` : ""}`, + ], + }); + const data = await chunk.stdout(); + if (data) { + offset += Buffer.byteLength(data); + stdout.write(data); + } + }; + for (;;) { + await new Promise((resolve) => setTimeout(resolve, 10_000)); + try { + await emitNewOutput(262_144); + const status = await sandbox!.runCommand({ + cmd: "bash", + args: ["-c", `[ -f ${exitPath} ] && cat ${exitPath} || echo RUNNING`], + }); + const text = (await status.stdout()).trim(); + pollFailures = 0; + if (text === "RUNNING") continue; + // Output written between the last chunk and completion is still in + // the file; drain it before reporting the outcome. + await emitNewOutput(); + if (text === "0") return; + throw new Error(`step "${step}" exited with code ${text}`); + } catch (err) { + if (err instanceof Error && err.message.startsWith(`step "`)) throw err; + // Transient control-plane hiccups shouldn't kill a healthy run. + pollFailures += 1; + if (pollFailures >= 6) throw err; + stderr.write( + `poll failed (${pollFailures}/6), retrying: ${err instanceof Error ? err.message : err}\n`, + ); + } + } + }; + + // -- The actual matrix job: same command line as eval-refresh.yml. + const evalArgs = [ + `--experiment "${pair.experiment}"`, + pair.experimentSuite && `--experiment-suite "${pair.experimentSuite}"`, + `--eval "${pair.evalId}"`, + `--runs ${options.runs}`, + `--timeout-sec ${options.timeoutSec}`, + ] + .filter(Boolean) + .join(" "); + await runPolled("run eval", `pnpm eval -- ${evalArgs}`); + + // -- Collect: the artifact-upload equivalent. The results tree (scores, + // transcripts, and the attempt workspaces run-eval exports under + // results///attempt-N/workspace) is tarred into one + // archive and streamed out with downloadFile — SDK file retrieval is + // per-file, and streaming to disk avoids buffering workspaces in memory. + await run("pack results", "tar -czf /tmp/eval-results.tgz -C results ."); + const staging = mkdtempSync(join(tmpdir(), "vercel-eval-results-")); + try { + const archivePath = join(staging, "results.tgz"); + const downloaded = await sandbox.downloadFile( + { path: "/tmp/eval-results.tgz" }, + { path: archivePath }, + ); + if (!downloaded) throw new Error("results archive missing from sandbox"); + mkdirSync(options.resultsDir, { recursive: true }); + await execFileAsync("tar", [ + "-xzf", + archivePath, + "-C", + options.resultsDir, + ]); + } finally { + rmSync(staging, { recursive: true, force: true }); + } + log(`results extracted to ${options.resultsDir}`); + + return { + pair, + ok: true, + durationMs: Date.now() - start, + sandboxId: sandbox.name, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + stderr.write(`job failed: ${message}\n`); + return { + pair, + ok: false, + error: message, + durationMs: Date.now() - start, + sandboxId: sandbox?.name, + }; + } finally { + if (sandbox) { + try { + await sandbox.stop(); + log("sandbox stopped"); + } catch (err) { + stderr.write( + `sandbox stop failed (it will expire on its own): ${err instanceof Error ? err.message : String(err)}\n`, + ); + } + } + stdout.end(); + stderr.end(); + } +} diff --git a/packages/vercel-runner/tsconfig.json b/packages/vercel-runner/tsconfig.json new file mode 100644 index 00000000..564a5990 --- /dev/null +++ b/packages/vercel-runner/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 884776d6..df306dbf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -348,6 +348,22 @@ importers: specifier: 'catalog:' version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + packages/vercel-runner: + dependencies: + '@vercel/sandbox': + specifier: ^2.8.0 + version: 2.8.0 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 22.19.20 + tsx: + specifier: ^4.19.0 + version: 4.22.4 + typescript: + specifier: 'catalog:' + version: 5.9.3 + packages: '@adobe/css-tools@4.5.0': @@ -2300,6 +2316,9 @@ packages: resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} engines: {node: '>= 20'} + '@vercel/sandbox@2.8.0': + resolution: {integrity: sha512-1gmIXWgueeBBwBitVl1xqOsYmVv7xr7qX1qXeqwLGZTKGG1SkkpYkDzpWZ8LIDN0PmPGCqE6lvYao3zP8iz5uw==} + '@vitejs/plugin-react@5.2.0': resolution: {integrity: sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2335,6 +2354,9 @@ packages: '@vitest/utils@4.1.8': resolution: {integrity: sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==} + '@workflow/serde@4.1.0-beta.2': + resolution: {integrity: sha512-8kkeoQKLDaKXefjV5dbhBj2aErfKp1Mc4pb6tj8144cF+Em5SPbyMbyLCHp+BVrFfFVCBluCtMx+jjvaFVZGww==} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -2423,6 +2445,17 @@ packages: resolution: {integrity: sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==} engines: {node: '>=4'} + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + + b4a@1.8.1: + resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -2430,6 +2463,14 @@ packages: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} + bare-events@2.9.1: + resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + baseline-browser-mapping@2.10.35: resolution: {integrity: sha512-honAfLBde0HAFLdNyBEfuuENkF6zR+ozxqxa/2zJKHBe1qzLqyTSeRKpdPEHAP03rlDGyQOPnCSxnVpVqQo9Mg==} engines: {node: '>=6.0.0'} @@ -2823,6 +2864,9 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + eventsource-parser@3.1.0: resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} engines: {node: '>=18.0.0'} @@ -2905,6 +2949,9 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -3290,6 +3337,9 @@ packages: jsonfile@6.2.1: resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} + jsonlines@0.1.1: + resolution: {integrity: sha512-ekDrAGso79Cvf+dtm+mL8OBI2bmAOt3gssYs833De/C9NmIpWDWyUO4zPgB5x2/OhY366dkhgfPMYfwZF7yOZA==} + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -3601,6 +3651,10 @@ packages: resolution: {integrity: sha512-weP+BZ8MVNnlCm8c0Qdc1WSWq4Qn7I+9CJGm7Qali6g44e/PUzbjNqJX5NJ9ljlNMosfJvg1fKEGILklK9cwnw==} engines: {node: '>=18'} + os-paths@4.4.0: + resolution: {integrity: sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==} + engines: {node: '>= 6.0'} + p-limit@3.1.0: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} @@ -3932,6 +3986,10 @@ packages: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -4055,6 +4113,9 @@ packages: resolution: {integrity: sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==} engines: {node: '>=18'} + streamx@2.28.0: + resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} + strfy-js@3.2.2: resolution: {integrity: sha512-hUgJ5k2PR1ivhq4uObxnin5j6GcOr0Y0N1lzi3z6SRhxNqu4rzpDfyoC2ToUAyM8yXNXM0zs6f4KIiqj8NqheQ==} @@ -4116,6 +4177,12 @@ packages: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + + text-decoder@1.2.7: + resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} + tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} @@ -4199,6 +4266,10 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici@7.28.0: + resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} + engines: {node: '>=20.18.1'} + unicorn-magic@0.3.0: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} @@ -4385,6 +4456,14 @@ packages: resolution: {integrity: sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==} engines: {node: '>=20'} + xdg-app-paths@5.1.0: + resolution: {integrity: sha512-RAQ3WkPf4KTU1A8RtFx3gWywzVKe00tfOPFfl2NDGqbIFENQO4kqAJp7mhQjNj/33W5x5hiWWUdyfPq/5SU3QA==} + engines: {node: '>=6'} + + xdg-portable@7.3.0: + resolution: {integrity: sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw==} + engines: {node: '>= 6.0'} + xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -6393,6 +6472,23 @@ snapshots: '@vercel/oidc@3.2.0': {} + '@vercel/sandbox@2.8.0': + dependencies: + '@vercel/oidc': 3.2.0 + '@workflow/serde': 4.1.0-beta.2 + async-retry: 1.3.3 + jose: 6.2.3 + jsonlines: 0.1.1 + ms: 2.1.3 + picocolors: 1.1.1 + tar-stream: 3.1.7 + undici: 7.28.0 + xdg-app-paths: 5.1.0 + zod: 4.4.3 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + '@vitejs/plugin-react@5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))': dependencies: '@babel/core': 7.29.7 @@ -6454,6 +6550,8 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.0 + '@workflow/serde@4.1.0-beta.2': {} + accepts@2.0.0: dependencies: mime-types: 3.0.2 @@ -6531,10 +6629,18 @@ snapshots: dependencies: tslib: 2.8.1 + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + + b4a@1.8.1: {} + balanced-match@1.0.2: {} balanced-match@4.0.4: {} + bare-events@2.9.1: {} + baseline-browser-mapping@2.10.35: {} bcryptjs@3.0.3: {} @@ -6934,6 +7040,12 @@ snapshots: etag@1.8.1: {} + events-universal@1.0.1: + dependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - bare-abort-controller + eventsource-parser@3.1.0: {} eventsource@3.0.7: @@ -7048,6 +7160,8 @@ snapshots: fast-deep-equal@3.1.3: {} + fast-fifo@1.3.2: {} + fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -7374,6 +7488,8 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 + jsonlines@0.1.1: {} + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -7630,6 +7746,8 @@ snapshots: string-width: 7.2.0 strip-ansi: 7.2.0 + os-paths@4.4.0: {} + p-limit@3.1.0: dependencies: yocto-queue: 0.1.0 @@ -7956,6 +8074,8 @@ snapshots: onetime: 7.0.0 signal-exit: 4.1.0 + retry@0.13.1: {} + reusify@1.1.0: {} rollup@4.61.1: @@ -8149,6 +8269,15 @@ snapshots: stdin-discarder@0.2.2: {} + streamx@2.28.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.7 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + strfy-js@3.2.2: dependencies: minimatch: 10.2.5 @@ -8199,6 +8328,21 @@ snapshots: tapable@2.3.3: {} + tar-stream@3.1.7: + dependencies: + b4a: 1.8.1 + fast-fifo: 1.3.2 + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + text-decoder@1.2.7: + dependencies: + b4a: 1.8.1 + transitivePeerDependencies: + - react-native-b4a + tiny-invariant@1.3.3: {} tinybench@2.9.0: {} @@ -8274,6 +8418,8 @@ snapshots: undici-types@7.24.6: {} + undici@7.28.0: {} + unicorn-magic@0.3.0: {} universalify@2.0.1: {} @@ -8431,6 +8577,14 @@ snapshots: is-wsl: 3.1.1 powershell-utils: 0.1.0 + xdg-app-paths@5.1.0: + dependencies: + xdg-portable: 7.3.0 + + xdg-portable@7.3.0: + dependencies: + os-paths: 4.4.0 + xtend@4.0.2: {} yallist@3.1.1: {} From dba1398053dff1b4ecd51f7624fe1fdccbe7a88c Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 23 Jul 2026 20:31:00 +0100 Subject: [PATCH 2/9] feat: add eval-refresh-vercel.yml, a Vercel Sandbox twin of the matrix A single dispatcher job (pnpm eval:vercel) with the same inputs, gating, and publish steps as eval-refresh.yml, which stays untouched. Triggered by manual dispatch or the new run-evals-sandbox / run-evals-sandbox-changed PR labels; own concurrency group and results-PR branch. Validated in CI: dispatch runs on haiku and sonnet (job body byte-identical, run under the eval-refresh.yml name pre-move) and a live label-triggered run on PR #114. Co-Authored-By: Claude Fable 5 --- .github/workflows/eval-refresh-vercel.yml | 312 ++++++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 .github/workflows/eval-refresh-vercel.yml diff --git a/.github/workflows/eval-refresh-vercel.yml b/.github/workflows/eval-refresh-vercel.yml new file mode 100644 index 00000000..830cb515 --- /dev/null +++ b/.github/workflows/eval-refresh-vercel.yml @@ -0,0 +1,312 @@ +name: Refresh eval results (Vercel Sandbox) + +# The Vercel Sandbox counterpart of eval-refresh.yml (AI-912 spike): identical +# inputs and publish behavior, but instead of a runner-per-pair Actions matrix, +# a single job runs `pnpm eval:vercel` (packages/vercel-runner), which +# discovers the same experiment/eval pairs, runs each in its own Vercel +# Sandbox microVM in parallel, collects results/ back, and runs the same +# export step. The Docker work (agent container + Supabase sibling containers) +# happens inside each microVM's own dockerd. +# +# Triggers: manual dispatch, or the run-evals-sandbox / run-evals-sandbox-changed +# PR labels — the sandbox counterparts of eval-refresh.yml's run-evals / +# run-evals-changed. To promote this to the primary workflow, move the +# `schedule` trigger over from eval-refresh.yml. + +on: + workflow_dispatch: + inputs: + experiments: + description: "Comma-separated experiment names to run (blank to auto-discover from experiment_suite)" + required: false + default: "" + eval: + description: "Optional comma-separated eval ids to run" + required: false + default: "" + suite: + description: "Comma-separated eval suites to run" + required: true + default: "benchmark" + experiment_suite: + description: "Comma-separated experiment suites to run" + required: true + default: "benchmark,no-skills" + runs: + description: "Attempts per experiment/eval pair" + required: true + default: "2" + timeout_sec: + description: "Timeout per attempt in seconds" + required: true + default: "720" + merge: + description: "Merge into existing results instead of overwriting (graft new experiment/eval pairs)" + type: boolean + required: false + default: false + commit_to_branch: + description: "Commit exported results to the dispatched branch instead of opening a PR" + type: boolean + required: false + default: false + pull_request: + # Run whenever a PR carrying the run-evals-sandbox label is opened, pushed + # to, or receives the label. The job-level `if` gates on those cases. + types: [opened, synchronize, labeled] + +permissions: + contents: write + pull-requests: write + actions: read + +concurrency: + group: eval-refresh-vercel-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} + # Supersede an in-flight run when a new commit is pushed to the same PR, but + # never cancel a workflow_dispatch run (those open the refresh PR). + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + run-evals: + if: >- + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + (github.event_name == 'pull_request' && + (contains(github.event.pull_request.labels.*.name, 'run-evals-sandbox') || + contains(github.event.pull_request.labels.*.name, 'run-evals-sandbox-changed')) && + github.event.pull_request.head.repo.full_name == github.repository && + (github.event.action != 'labeled' || + github.event.label.name == 'run-evals-sandbox' || + github.event.label.name == 'run-evals-sandbox-changed')) + runs-on: ubuntu-latest + steps: + - name: Prepare inputs + id: inputs + shell: bash + run: | + set -euo pipefail + + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + experiments="${{ inputs.experiments }}" + eval_ids="${{ inputs.eval }}" + suite="${{ inputs.suite }}" + experiment_suite="${{ inputs.experiment_suite }}" + runs="${{ inputs.runs }}" + timeout_sec="${{ inputs.timeout_sec }}" + elif [ "${{ github.event_name }}" = "schedule" ]; then + experiments="" + eval_ids="" + suite="regression" + experiment_suite="regression" + runs="2" + timeout_sec="720" + else + experiments="" + eval_ids="" + suite="benchmark,regression" + experiment_suite="benchmark,no-skills,regression" + runs="2" + timeout_sec="720" + fi + + # run-evals-sandbox takes priority; run-evals-sandbox-changed only + # filters when it is absent. filter_changed drives the changed-eval + # filter (PR-only, needs a PR diff). + filter_changed="false" + if [ "${{ github.event_name }}" = "pull_request" ] && \ + [ "${{ contains(github.event.pull_request.labels.*.name, 'run-evals-sandbox-changed') }}" = "true" ] && \ + [ "${{ contains(github.event.pull_request.labels.*.name, 'run-evals-sandbox') }}" = "false" ]; then + filter_changed="true" + fi + + # do_merge drives the export --merge (graft into existing results). It's + # always on for the changed path, and opt-in for manual dispatch. + do_merge="$filter_changed" + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.merge }}" = "true" ]; then + do_merge="true" + fi + + { + echo "experiments=$experiments" + echo "eval=$eval_ids" + echo "suite=$suite" + echo "experiment_suite=$experiment_suite" + echo "runs=$runs" + echo "timeout_sec=$timeout_sec" + echo "filter_changed=$filter_changed" + echo "do_merge=$do_merge" + } >> "$GITHUB_OUTPUT" + + - name: Generate GitHub App token + id: generate-token + # A GitHub App token is needed so the push below can trigger the + # gh-pages workflow: GITHUB_TOKEN pushes don't trigger other workflows. + # https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow#triggering-a-workflow-from-a-workflow + if: >- + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch) + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + submodules: recursive + ref: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref }} + token: ${{ steps.generate-token.outputs.token || github.token }} + + - name: Install pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Determine changed evals + id: changed + if: steps.inputs.outputs.filter_changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + + suites=",${{ steps.inputs.outputs.suite }}," + changed=() + while IFS= read -r id; do + # A deleted eval dir still shows in the diff but can't be run, and + # evals outside the requested suites (e.g. `other`) don't run in CI. + [ -f "evals/$id/PROMPT.md" ] || continue + suite_val=$(sed -n 's/^suite:[[:space:]]*//p' "evals/$id/PROMPT.md" | head -n 1) + case "$suites" in *",$suite_val,"*) changed+=("$id") ;; esac + done < <(gh pr diff ${{ github.event.pull_request.number }} --name-only \ + | grep '^evals/' | cut -d/ -f2 | sort -u || true) + + if [ "${#changed[@]}" -eq 0 ]; then + echo "No matching eval directories changed in this PR — nothing to run" + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Changed eval dirs: ${changed[*]}" + echo "skip=false" >> "$GITHUB_OUTPUT" + echo "eval=$(IFS=,; echo "${changed[*]}")" >> "$GITHUB_OUTPUT" + + - name: Run evals in Vercel Sandbox + if: steps.changed.outputs.skip != 'true' + shell: bash + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + # The sandboxes clone this (internal) repo over HTTPS. + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + args=( + --revision "$(git rev-parse HEAD)" + --suite "${{ steps.inputs.outputs.suite }}" + --experiment-suite "${{ steps.inputs.outputs.experiment_suite }}" + --runs "${{ steps.inputs.outputs.runs }}" + --timeout-sec "${{ steps.inputs.outputs.timeout_sec }}" + --concurrency 8 + ) + + experiments="${{ steps.inputs.outputs.experiments }}" + [ -n "$experiments" ] && args+=(--experiment "$experiments") + + # The changed-eval filter (when active) narrows the eval list. + eval_ids="${{ steps.changed.outputs.eval || steps.inputs.outputs.eval }}" + [ -n "$eval_ids" ] && args+=(--eval "$eval_ids") + + [ "${{ steps.inputs.outputs.do_merge }}" = "true" ] && args+=(--merge) + + pnpm eval:vercel -- "${args[@]}" + + - name: Upload raw results + if: steps.changed.outputs.skip != 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: raw-results + path: results/ + retention-days: 3 + + - name: Upload exported results + if: >- + steps.changed.outputs.skip != 'true' && + (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eval-results-json + path: apps/web/src/data/*eval-results.json + if-no-files-found: error + retention-days: 7 + + - name: Commit exported results to branch + # PR runs commit to the PR head branch. Manual dispatch commits to the + # selected branch only when commit_to_branch is enabled. Scheduled runs + # go through a PR instead (see "Create results pull request" below): + # main's branch protection rejects a direct push. + if: >- + steps.changed.outputs.skip != 'true' && + (github.event_name == 'pull_request' || + (github.event_name == 'workflow_dispatch' && inputs.commit_to_branch)) + shell: bash + run: | + set -euo pipefail + + git config user.name "github-actions[bot]" + # github-actions[bot]'s noreply email uses its public user ID: https://github.com/actions/checkout#push-a-commit-using-the-built-in-token + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + for result_file in apps/web/src/data/*eval-results.json; do + [ -f "$result_file" ] && git add "$result_file" + done + + if git diff --cached --quiet; then + echo "No eval result changes to commit" + exit 0 + fi + + git commit -m "chore: refresh eval results" + git push + + - name: Create results pull request + if: >- + steps.changed.outputs.skip != 'true' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch)) + id: cpr + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.generate-token.outputs.token }} + add-paths: apps/web/src/data/*eval-results.json + # Per-ref-and-event head branch so a branch refresh PR can't collide + # with another ref's, with the scheduled run's, or with + # eval-refresh.yml's. + branch: chore/refresh-eval-results-vercel-${{ github.ref_name }}-${{ github.event_name }} + base: ${{ github.ref_name }} + commit-message: "chore: refresh eval results" + title: "chore: refresh eval results" + body: | + Refreshes `apps/web/src/data/eval-results.json` from the latest automated eval run. + # Draft PRs can't be merged, and the scheduled path merges itself. + draft: ${{ github.event_name != 'schedule' }} + delete-branch: true + + - name: Merge scheduled results pull request + # The app is on the ruleset's bypass list, so no review is required. + if: github.event_name == 'schedule' && steps.cpr.outputs.pull-request-number + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + run: gh pr merge "${{ steps.cpr.outputs.pull-request-number }}" --squash --delete-branch From 4c07a5e80da0ce02b5a7bec851416502a6732ce2 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 23 Jul 2026 23:16:20 +0100 Subject: [PATCH 3/9] fix: write .env before dispatch; widen and parameterize concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Experiment discovery (pnpm eval -- list) shells into framework scripts whose --env-file hard-requires a repo-root .env — the full-matrix label path crashed in CI without it. Write the same .env the old run-evals job wrote. Also: concurrency becomes a dispatch input (default 16 — the matrix fanned as wide as the runner pool), sandbox creation retries on a short backoff (rate limits at full-matrix scale), and raw results upload with !cancelled() so passing pairs survive a failed sibling, like per-job matrix artifacts did. Co-Authored-By: Claude Fable 5 --- .github/workflows/eval-refresh-vercel.yml | 34 +++++++++++++-- packages/vercel-runner/README.md | 6 ++- packages/vercel-runner/src/sandbox-job.ts | 52 ++++++++++++++++------- 3 files changed, 71 insertions(+), 21 deletions(-) diff --git a/.github/workflows/eval-refresh-vercel.yml b/.github/workflows/eval-refresh-vercel.yml index 830cb515..a423485c 100644 --- a/.github/workflows/eval-refresh-vercel.yml +++ b/.github/workflows/eval-refresh-vercel.yml @@ -50,6 +50,10 @@ on: type: boolean required: false default: false + concurrency: + description: "Max sandboxes in flight at once" + required: false + default: "16" pull_request: # Run whenever a PR carrying the run-evals-sandbox label is opened, pushed # to, or receives the label. The job-level `if` gates on those cases. @@ -93,6 +97,7 @@ jobs: experiment_suite="${{ inputs.experiment_suite }}" runs="${{ inputs.runs }}" timeout_sec="${{ inputs.timeout_sec }}" + concurrency="${{ inputs.concurrency }}" elif [ "${{ github.event_name }}" = "schedule" ]; then experiments="" eval_ids="" @@ -100,6 +105,7 @@ jobs: experiment_suite="regression" runs="2" timeout_sec="720" + concurrency="16" else experiments="" eval_ids="" @@ -107,6 +113,7 @@ jobs: experiment_suite="benchmark,no-skills,regression" runs="2" timeout_sec="720" + concurrency="16" fi # run-evals-sandbox takes priority; run-evals-sandbox-changed only @@ -133,6 +140,7 @@ jobs: echo "experiment_suite=$experiment_suite" echo "runs=$runs" echo "timeout_sec=$timeout_sec" + echo "concurrency=$concurrency" echo "filter_changed=$filter_changed" echo "do_merge=$do_merge" } >> "$GITHUB_OUTPUT" @@ -199,12 +207,27 @@ jobs: echo "skip=false" >> "$GITHUB_OUTPUT" echo "eval=$(IFS=,; echo "${changed[*]}")" >> "$GITHUB_OUTPUT" - - name: Run evals in Vercel Sandbox - if: steps.changed.outputs.skip != 'true' + - name: Write eval environment + # The framework's `pnpm eval` scripts load the repo-root .env with + # node's --env-file, which hard-fails when the file is missing — and + # experiment discovery (`pnpm eval -- list`) shells out to them. Same + # step as eval-refresh.yml's run-evals job. shell: bash env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set -euo pipefail + + { + echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" + echo "OPENAI_API_KEY=${OPENAI_API_KEY}" + } > .env + + - name: Run evals in Vercel Sandbox + if: steps.changed.outputs.skip != 'true' + shell: bash + env: VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} @@ -219,7 +242,7 @@ jobs: --experiment-suite "${{ steps.inputs.outputs.experiment_suite }}" --runs "${{ steps.inputs.outputs.runs }}" --timeout-sec "${{ steps.inputs.outputs.timeout_sec }}" - --concurrency 8 + --concurrency "${{ steps.inputs.outputs.concurrency }}" ) experiments="${{ steps.inputs.outputs.experiments }}" @@ -234,7 +257,10 @@ jobs: pnpm eval:vercel -- "${args[@]}" - name: Upload raw results - if: steps.changed.outputs.skip != 'true' + # !cancelled(): with the matrix, pairs that passed still uploaded their + # artifacts when a sibling job failed; keep partial results retrievable + # the same way when one sandbox job fails mid-fan-out. + if: ${{ !cancelled() && steps.changed.outputs.skip != 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: raw-results diff --git a/packages/vercel-runner/README.md b/packages/vercel-runner/README.md index f823d99a..ade52b7f 100644 --- a/packages/vercel-runner/README.md +++ b/packages/vercel-runner/README.md @@ -38,7 +38,7 @@ This runner replaces step 2's machine, one Firecracker microVM per pair: | `ubuntu-latest` runner | `Sandbox.create()` VM (Amazon Linux 2023) | | `actions/checkout` + submodules | `source: { type: "git", revision }` + submodule init | | runner's built-in Docker daemon | `dnf install docker` + detached `dockerd` | -| `strategy.matrix` parallelism | N sandboxes in flight (`--concurrency`) | +| `strategy.matrix` parallelism | N sandboxes in flight (`--concurrency`; the workflow defaults to 16 — the matrix ran as wide as the org's runner pool, ~20–60 jobs, while Vercel Pro allows 2,000 concurrent sandboxes and 50 × 4-vCPU creations/min) | | artifact upload/download | `tar` + `sandbox.readFileToBuffer()` → `results/` | `packages/sandbox` is untouched: the agent container and the Supabase sibling @@ -56,7 +56,9 @@ pnpm eval:vercel -- \ Omit `--experiment`/`--eval` to fan out over the same matrix the scheduled workflow would (`--suite`, `--experiment-suite` filter it). `--dry` prints the -plan without dispatching. The sandbox runs the **pushed commit** (`--revision` +plan without dispatching. Experiment discovery shells out to `pnpm eval -- +list`, whose `--env-file` hard-requires a repo-root `.env` — the file must +exist (even empty; the workflow writes it like eval-refresh.yml does). The sandbox runs the **pushed commit** (`--revision` overrides; defaults to `HEAD`, which must be on a remote branch). Required in `.env`: diff --git a/packages/vercel-runner/src/sandbox-job.ts b/packages/vercel-runner/src/sandbox-job.ts index 66798e6c..b515fa49 100644 --- a/packages/vercel-runner/src/sandbox-job.ts +++ b/packages/vercel-runner/src/sandbox-job.ts @@ -28,6 +28,15 @@ const SANDBOX_CWD = "/vercel/sandbox"; /** Ready-poll budget for dockerd inside the VM. */ const DOCKERD_READY_TIMEOUT_SEC = 60; +/** + * Backoff schedule for retrying sandbox creation. A full-matrix fan-out can + * brush the vCPU allocation rate limit or hit a transient API error; like the + * sandbox-image build retry in packages/sandbox, every failure is retried (no + * stable error taxonomy) and a deterministic failure merely wastes the + * schedule before surfacing. + */ +const CREATE_RETRY_DELAYS_MS = [20_000, 60_000]; + export interface SandboxJobOptions { pair: EvalPair; /** HTTPS clone URL of this repo. */ @@ -105,21 +114,34 @@ export async function runPairInSandbox( let sandbox: Sandbox | undefined; try { log(`creating sandbox (vcpus=${options.vcpus}, timeout=${Math.round(options.sandboxTimeoutMs / 60000)}m, rev=${options.revision.slice(0, 8)})`); - sandbox = await Sandbox.create({ - ...credentialsFromEnv(), - runtime: "node22", - resources: { vcpus: options.vcpus }, - timeout: options.sandboxTimeoutMs, - source: { - type: "git", - url: options.repoUrl, - revision: options.revision, - // Vercel clones with these as basic-auth; x-access-token is GitHub's - // conventional username for token auth. - username: "x-access-token", - password: options.githubToken, - }, - }); + for (let attempt = 0; ; attempt++) { + try { + sandbox = await Sandbox.create({ + ...credentialsFromEnv(), + runtime: "node22", + resources: { vcpus: options.vcpus }, + timeout: options.sandboxTimeoutMs, + source: { + type: "git", + url: options.repoUrl, + revision: options.revision, + // Vercel clones with these as basic-auth; x-access-token is + // GitHub's conventional username for token auth. + username: "x-access-token", + password: options.githubToken, + }, + }); + break; + } catch (err) { + const delayMs = CREATE_RETRY_DELAYS_MS[attempt]; + if (delayMs === undefined) throw err; + stderr.write( + `sandbox creation failed (attempt ${attempt + 1}/${CREATE_RETRY_DELAYS_MS.length + 1}), ` + + `retrying in ${delayMs / 1000}s: ${err instanceof Error ? err.message : err}\n`, + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } log(`sandbox ${sandbox.name} created`); const run = async ( From 90124a8f1476dfc16f7501bbf9f87741f6515ee1 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 23 Jul 2026 23:21:35 +0100 Subject: [PATCH 4/9] chore: widen sandbox fan-out to 32 (Pro-plan creation rate has headroom) Co-Authored-By: Claude Fable 5 --- .github/workflows/eval-refresh-vercel.yml | 6 +++--- packages/vercel-runner/README.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/eval-refresh-vercel.yml b/.github/workflows/eval-refresh-vercel.yml index a423485c..897f4270 100644 --- a/.github/workflows/eval-refresh-vercel.yml +++ b/.github/workflows/eval-refresh-vercel.yml @@ -53,7 +53,7 @@ on: concurrency: description: "Max sandboxes in flight at once" required: false - default: "16" + default: "32" pull_request: # Run whenever a PR carrying the run-evals-sandbox label is opened, pushed # to, or receives the label. The job-level `if` gates on those cases. @@ -105,7 +105,7 @@ jobs: experiment_suite="regression" runs="2" timeout_sec="720" - concurrency="16" + concurrency="32" else experiments="" eval_ids="" @@ -113,7 +113,7 @@ jobs: experiment_suite="benchmark,no-skills,regression" runs="2" timeout_sec="720" - concurrency="16" + concurrency="32" fi # run-evals-sandbox takes priority; run-evals-sandbox-changed only diff --git a/packages/vercel-runner/README.md b/packages/vercel-runner/README.md index ade52b7f..983d5ef2 100644 --- a/packages/vercel-runner/README.md +++ b/packages/vercel-runner/README.md @@ -38,8 +38,8 @@ This runner replaces step 2's machine, one Firecracker microVM per pair: | `ubuntu-latest` runner | `Sandbox.create()` VM (Amazon Linux 2023) | | `actions/checkout` + submodules | `source: { type: "git", revision }` + submodule init | | runner's built-in Docker daemon | `dnf install docker` + detached `dockerd` | -| `strategy.matrix` parallelism | N sandboxes in flight (`--concurrency`; the workflow defaults to 16 — the matrix ran as wide as the org's runner pool, ~20–60 jobs, while Vercel Pro allows 2,000 concurrent sandboxes and 50 × 4-vCPU creations/min) | -| artifact upload/download | `tar` + `sandbox.readFileToBuffer()` → `results/` | +| `strategy.matrix` parallelism | N sandboxes in flight (`--concurrency`; the workflow defaults to 32 — the matrix ran as wide as the org's runner pool, ~20–60 jobs, while Vercel Pro allows 2,000 concurrent sandboxes and 50 × 4-vCPU creations/min) | +| artifact upload/download | `tar` + `sandbox.downloadFile()` → `results/` | `packages/sandbox` is untouched: the agent container and the Supabase sibling containers run against the VM's own dockerd exactly as they do against a From 025c9218e6124e028237072472c95b27950d3c9c Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 23 Jul 2026 23:55:17 +0100 Subject: [PATCH 5/9] feat: fleet progress heartbeat, scored verdicts, dashboard tags A once-a-minute scoreboard line (done/pass/fail/error, per-phase in-flight counts, longest-running pair) keeps wide CI fan-outs legible; the summary now shows each pair's scored eval verdict (job ok != eval pass); sandboxes carry run/experiment/eval tags so the Vercel dashboard filters a CI run's fleet live. Co-Authored-By: Claude Fable 5 --- packages/vercel-runner/src/run.ts | 52 +++++++++++++++++++++-- packages/vercel-runner/src/sandbox-job.ts | 37 +++++++++++++++- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/packages/vercel-runner/src/run.ts b/packages/vercel-runner/src/run.ts index 15897576..e286ef03 100644 --- a/packages/vercel-runner/src/run.ts +++ b/packages/vercel-runner/src/run.ts @@ -17,13 +17,22 @@ import { execFileSync, spawnSync } from "node:child_process"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { discoverPairs, type EvalPair } from "./discover.js"; -import { runPairInSandbox, type SandboxJobResult } from "./sandbox-job.js"; +import { + runPairInSandbox, + type SandboxJobPhase, + type SandboxJobResult, +} from "./sandbox-job.js"; const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); /** Setup (docker + pnpm install) and scoring headroom around the eval runs. */ const SANDBOX_SETUP_HEADROOM_MIN = 20; +/** Fleet progress heartbeat interval — one scoreboard line per minute. */ +const HEARTBEAT_INTERVAL_MS = 60_000; + +const PHASE_ORDER: SandboxJobPhase[] = ["create", "bootstrap", "eval", "collect"]; + const rawArgs = process.argv.slice(2).filter((arg) => arg !== "--"); function readFlag(name: string): string | undefined { @@ -133,14 +142,42 @@ async function main() { // own isolated machine; only the number in flight at once is capped. const queue: EvalPair[] = [...pairs]; const results: SandboxJobResult[] = []; + const inFlight = new Map(); + const startedAt = Date.now(); + + // Once-a-minute scoreboard so a wide fan-out stays legible in CI logs: + // completions, per-phase in-flight counts, and the longest-running pair + // (the one to look at when something hangs). + const heartbeat = setInterval(() => { + if (results.length === pairs.length) return; + const doneOk = results.filter((result) => result.ok).length; + const passed = results.filter((result) => result.evalPassed === true).length; + const failed = results.filter((result) => result.evalPassed === false).length; + const phases = PHASE_ORDER.map( + (phase) => + `${[...inFlight.values()].filter((state) => state.phase === phase).length} ${phase}`, + ).join(" · "); + const oldest = [...inFlight.entries()].sort((a, b) => a[1].since - b[1].since)[0]; + const oldestNote = oldest + ? ` · oldest: ${oldest[0]} in ${oldest[1].phase} ${Math.round((Date.now() - oldest[1].since) / 60000)}m` + : ""; + console.log( + `[progress ${Math.round((Date.now() - startedAt) / 60000)}m] ` + + `${results.length}/${pairs.length} done (${passed} pass · ${failed} fail · ${results.length - doneOk} error) · ` + + `in flight: ${phases} · ${queue.length} queued${oldestNote}`, + ); + }, HEARTBEAT_INTERVAL_MS); + const workers = Array.from( { length: Math.min(concurrency, pairs.length) }, async () => { let pair: EvalPair | undefined; while ((pair = queue.shift()) !== undefined) { + const key = `${pair.experiment} x ${pair.evalId}`; results.push( await runPairInSandbox({ pair, + onPhase: (phase) => inFlight.set(key, { phase, since: Date.now() }), repoUrl, revision, githubToken, @@ -152,17 +189,26 @@ async function main() { resultsDir: join(ROOT, "results"), }), ); + inFlight.delete(key); } }, ); - await Promise.all(workers); + try { + await Promise.all(workers); + } finally { + clearInterval(heartbeat); + } console.log("\n=== summary ==="); for (const result of results) { const minutes = (result.durationMs / 60000).toFixed(1); + const verdict = + result.evalPassed === undefined + ? "" + : `, eval ${result.evalPassed ? "PASS" : "FAIL"}`; console.log( `${result.ok ? "✅" : "💥"} ${result.pair.experiment} x ${result.pair.evalId} ` + - `(${minutes}m${result.ok ? "" : `, ${result.error}`})`, + `(${minutes}m${verdict}${result.ok ? "" : `, ${result.error}`})`, ); } const failed = results.filter((result) => !result.ok); diff --git a/packages/vercel-runner/src/sandbox-job.ts b/packages/vercel-runner/src/sandbox-job.ts index b515fa49..3766d7ec 100644 --- a/packages/vercel-runner/src/sandbox-job.ts +++ b/packages/vercel-runner/src/sandbox-job.ts @@ -11,7 +11,7 @@ * daemon runs on does. */ -import { mkdirSync, rmSync, mkdtempSync } from "node:fs"; +import { mkdirSync, rmSync, mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Writable } from "node:stream"; @@ -37,8 +37,13 @@ const DOCKERD_READY_TIMEOUT_SEC = 60; */ const CREATE_RETRY_DELAYS_MS = [20_000, 60_000]; +/** Coarse lifecycle phase of a pair's sandbox, for fleet progress reporting. */ +export type SandboxJobPhase = "create" | "bootstrap" | "eval" | "collect"; + export interface SandboxJobOptions { pair: EvalPair; + /** Phase-transition callback for the dispatcher's progress heartbeat. */ + onPhase?: (phase: SandboxJobPhase) => void; /** HTTPS clone URL of this repo. */ repoUrl: string; /** Commit SHA to run — must be reachable on the remote. */ @@ -79,6 +84,8 @@ export interface SandboxJobResult { ok: boolean; /** Failure summary when ok is false. */ error?: string; + /** Scored eval outcome from the extracted result JSON, when readable. */ + evalPassed?: boolean; durationMs: number; sandboxId?: string; } @@ -114,6 +121,7 @@ export async function runPairInSandbox( let sandbox: Sandbox | undefined; try { log(`creating sandbox (vcpus=${options.vcpus}, timeout=${Math.round(options.sandboxTimeoutMs / 60000)}m, rev=${options.revision.slice(0, 8)})`); + options.onPhase?.("create"); for (let attempt = 0; ; attempt++) { try { sandbox = await Sandbox.create({ @@ -121,6 +129,14 @@ export async function runPairInSandbox( runtime: "node22", resources: { vcpus: options.vcpus }, timeout: options.sandboxTimeoutMs, + // Makes the Vercel dashboard's Sandboxes view a live per-CI-run + // board, filterable by these keys (max 5 tags). + tags: { + runner: "supabase-evals", + run: process.env.GITHUB_RUN_ID ?? "local", + experiment: pair.experiment, + eval: pair.evalId, + }, source: { type: "git", url: options.repoUrl, @@ -143,6 +159,7 @@ export async function runPairInSandbox( } } log(`sandbox ${sandbox.name} created`); + options.onPhase?.("bootstrap"); const run = async ( step: string, @@ -277,7 +294,9 @@ export async function runPairInSandbox( ] .filter(Boolean) .join(" "); + options.onPhase?.("eval"); await runPolled("run eval", `pnpm eval -- ${evalArgs}`); + options.onPhase?.("collect"); // -- Collect: the artifact-upload equivalent. The results tree (scores, // transcripts, and the attempt workspaces run-eval exports under @@ -305,9 +324,25 @@ export async function runPairInSandbox( } log(`results extracted to ${options.resultsDir}`); + // Surface the scored outcome (run-eval exits 0 on a scored FAIL, so job + // ok ≠ eval pass). Best-effort: a missing/renamed file just omits it. + let evalPassed: boolean | undefined; + try { + const result = JSON.parse( + readFileSync( + join(options.resultsDir, pair.experiment, `${pair.evalId}.json`), + "utf8", + ), + ) as { passed?: unknown }; + if (typeof result.passed === "boolean") evalPassed = result.passed; + } catch { + // No scored verdict available. + } + return { pair, ok: true, + evalPassed, durationMs: Date.now() - start, sandboxId: sandbox.name, }; From 713f4540d68d4af52d0d144addae7d9627738946 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 00:14:06 +0100 Subject: [PATCH 6/9] fix: poll every sandbox step; retry errored pairs; wider, rate-shaped fan-out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full-matrix run (166 pairs) failed on a single bootstrap step dying with 'Stream ended before command finished' — only the eval step had been moved off live log streams. All steps now run detached with polled status/log files. Pairs that end in a job error (never scored FAILs) get one fresh-sandbox retry: at matrix scale even a sub-percent transient rate would fail most runs. Fan-out defaults to 64 with sandbox creations spaced ~1.3s apart, ramping under Pro's 200 vCPUs/min allocation rate instead of bouncing off it. Co-Authored-By: Claude Fable 5 --- .github/workflows/eval-refresh-vercel.yml | 6 +- packages/vercel-runner/README.md | 2 +- packages/vercel-runner/src/run.ts | 20 ++- packages/vercel-runner/src/sandbox-job.ts | 149 ++++++++++++---------- 4 files changed, 101 insertions(+), 76 deletions(-) diff --git a/.github/workflows/eval-refresh-vercel.yml b/.github/workflows/eval-refresh-vercel.yml index 897f4270..9d87307b 100644 --- a/.github/workflows/eval-refresh-vercel.yml +++ b/.github/workflows/eval-refresh-vercel.yml @@ -53,7 +53,7 @@ on: concurrency: description: "Max sandboxes in flight at once" required: false - default: "32" + default: "64" pull_request: # Run whenever a PR carrying the run-evals-sandbox label is opened, pushed # to, or receives the label. The job-level `if` gates on those cases. @@ -105,7 +105,7 @@ jobs: experiment_suite="regression" runs="2" timeout_sec="720" - concurrency="32" + concurrency="64" else experiments="" eval_ids="" @@ -113,7 +113,7 @@ jobs: experiment_suite="benchmark,no-skills,regression" runs="2" timeout_sec="720" - concurrency="32" + concurrency="64" fi # run-evals-sandbox takes priority; run-evals-sandbox-changed only diff --git a/packages/vercel-runner/README.md b/packages/vercel-runner/README.md index 983d5ef2..25b9e8dc 100644 --- a/packages/vercel-runner/README.md +++ b/packages/vercel-runner/README.md @@ -38,7 +38,7 @@ This runner replaces step 2's machine, one Firecracker microVM per pair: | `ubuntu-latest` runner | `Sandbox.create()` VM (Amazon Linux 2023) | | `actions/checkout` + submodules | `source: { type: "git", revision }` + submodule init | | runner's built-in Docker daemon | `dnf install docker` + detached `dockerd` | -| `strategy.matrix` parallelism | N sandboxes in flight (`--concurrency`; the workflow defaults to 32 — the matrix ran as wide as the org's runner pool, ~20–60 jobs, while Vercel Pro allows 2,000 concurrent sandboxes and 50 × 4-vCPU creations/min) | +| `strategy.matrix` parallelism | N sandboxes in flight (`--concurrency`; the workflow defaults to 64, ramped at ~46 creations/min to stay under the 200 vCPUs/min allocation rate — the matrix ran as wide as the org's runner pool, ~20–60 jobs, while Vercel Pro allows 2,000 concurrent sandboxes and 50 × 4-vCPU creations/min) | | artifact upload/download | `tar` + `sandbox.downloadFile()` → `results/` | `packages/sandbox` is untouched: the agent container and the Supabase sibling diff --git a/packages/vercel-runner/src/run.ts b/packages/vercel-runner/src/run.ts index e286ef03..320b3699 100644 --- a/packages/vercel-runner/src/run.ts +++ b/packages/vercel-runner/src/run.ts @@ -174,9 +174,9 @@ async function main() { let pair: EvalPair | undefined; while ((pair = queue.shift()) !== undefined) { const key = `${pair.experiment} x ${pair.evalId}`; - results.push( - await runPairInSandbox({ - pair, + const dispatch = () => + runPairInSandbox({ + pair: pair!, onPhase: (phase) => inFlight.set(key, { phase, since: Date.now() }), repoUrl, revision, @@ -187,8 +187,18 @@ async function main() { sandboxTimeoutMs, agentEnv, resultsDir: join(ROOT, "results"), - }), - ); + }); + let result = await dispatch(); + // One fresh-sandbox retry for job errors (infra by definition — + // scored FAILs return ok and don't come through here). At matrix + // scale a fraction-of-a-percent transient rate would otherwise fail + // most full runs; the matrix equivalent was a manual re-run of the + // failed job. + if (!result.ok) { + console.log(`🔁 retrying ${key} in a fresh sandbox after: ${result.error}`); + result = await dispatch(); + } + results.push(result); inFlight.delete(key); } }, diff --git a/packages/vercel-runner/src/sandbox-job.ts b/packages/vercel-runner/src/sandbox-job.ts index 3766d7ec..d8058dea 100644 --- a/packages/vercel-runner/src/sandbox-job.ts +++ b/packages/vercel-runner/src/sandbox-job.ts @@ -37,6 +37,32 @@ const DOCKERD_READY_TIMEOUT_SEC = 60; */ const CREATE_RETRY_DELAYS_MS = [20_000, 60_000]; +/** Poll cadence for detached step commands (status + incremental log tail). */ +const STEP_POLL_INTERVAL_MS = 5_000; + +/** + * Minimum spacing between sandbox creations across the whole dispatch. Pro + * allocates at most 200 vCPUs/min; at 4 vCPUs per sandbox that's 50 + * creations/min, so ~1.3s spacing (≈46/min) ramps a wide fan-out cleanly + * instead of bouncing off 429s. Concurrency (sandboxes in flight) is + * effectively unlimited by comparison (2,000 on Pro). + */ +const CREATE_SPACING_MS = 1_300; +let nextCreateSlotAt = 0; + +async function waitForCreateSlot(): Promise { + for (;;) { + const now = Date.now(); + if (now >= nextCreateSlotAt) { + // Synchronous check-and-claim: no await between read and write, so + // concurrent jobs on the single event loop can't grab the same slot. + nextCreateSlotAt = now + CREATE_SPACING_MS; + return; + } + await new Promise((resolve) => setTimeout(resolve, nextCreateSlotAt - now)); + } +} + /** Coarse lifecycle phase of a pair's sandbox, for fleet progress reporting. */ export type SandboxJobPhase = "create" | "bootstrap" | "eval" | "collect"; @@ -124,6 +150,7 @@ export async function runPairInSandbox( options.onPhase?.("create"); for (let attempt = 0; ; attempt++) { try { + await waitForCreateSlot(); sandbox = await Sandbox.create({ ...credentialsFromEnv(), runtime: "node22", @@ -161,82 +188,31 @@ export async function runPairInSandbox( log(`sandbox ${sandbox.name} created`); options.onPhase?.("bootstrap"); + // Every step runs decoupled from any live log stream: launch detached + // with output going to a file in the VM, then poll with short commands + // (status sentinel + incremental tail). A command held on one streaming + // connection dies with "Stream ended before command finished" when that + // connection drops even though the command and the VM are fine — at + // full-matrix scale (~1,000 step commands per run) that's a certainty, + // and it killed bootstrap steps in practice, not just the long eval. + let stepCounter = 0; const run = async ( step: string, cmd: string, - opts: { sudo?: boolean; env?: Record } = {}, + opts: { sudo?: boolean } = {}, ) => { log(`--- ${step}`); - const result = await sandbox!.runCommand({ - cmd: "bash", - args: ["-c", `set -euo pipefail\n${cmd}`], - cwd: SANDBOX_CWD, - sudo: opts.sudo, - env: opts.env, - stdout, - stderr, - }); - if (result.exitCode !== 0) { - throw new Error(`step "${step}" exited with code ${result.exitCode}`); - } - }; - - // -- Bootstrap: what actions/checkout + setup-node + the runner image give - // us for free on GitHub Actions. - await run("install docker", "dnf install -y -q docker", { sudo: true }); - await sandbox.runCommand({ cmd: "dockerd", sudo: true, detached: true }); - await run( - "wait for dockerd", - `for i in $(seq ${DOCKERD_READY_TIMEOUT_SEC}); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1`, - { sudo: true }, - ); - // The harness shells out to `docker` as the unprivileged user; inside a - // single-tenant ephemeral VM, opening the socket is the simple safe way. - await run("open docker socket", "chmod 666 /var/run/docker.sock", { - sudo: true, - }); - - // The agent-skills submodule is public; rewrite its SSH URL to HTTPS since - // the VM has no GitHub SSH identity. The rewrite must live in --global - // config: the child `git clone` a submodule update spawns doesn't read the - // superproject's repo-local config. - await run( - "checkout submodules", - `git config --global url."https://github.com/".insteadOf "git@github.com:"\ngit submodule update --init --recursive`, - ); - - await run("install pnpm", "npm install -g pnpm@10.24.0", { sudo: true }); - await run("pnpm install", "pnpm install --frozen-lockfile"); - - // `pnpm eval` reads the repo-root .env (node --env-file). - await sandbox.writeFiles([ - { - path: ".env", - content: Buffer.from( - Object.entries(options.agentEnv) - .map(([key, value]) => `${key}=${value}`) - .join("\n") + "\n", - ), - }, - ]); - - // Long-running step, decoupled from any live log stream: launch detached - // with output going to a file in the VM, then poll with short commands - // (status sentinel + incremental tail). A multi-minute `pnpm eval` held on - // one streaming connection dies with "Stream ended before command - // finished" when that connection drops, even though the command and the - // VM are fine. - const runPolled = async (step: string, cmd: string) => { - log(`--- ${step}`); - const logPath = "/tmp/step.log"; - const exitPath = "/tmp/step.exit"; + const id = ++stepCounter; + const logPath = `/tmp/step-${id}.log`; + const exitPath = `/tmp/step-${id}.exit`; await sandbox!.runCommand({ cmd: "bash", args: [ "-c", - `rm -f ${logPath} ${exitPath}; (set -euo pipefail\n${cmd}\n) >${logPath} 2>&1; echo $? >${exitPath}`, + `(set -euo pipefail\n${cmd}\n) >${logPath} 2>&1; echo $? >${exitPath}`, ], cwd: SANDBOX_CWD, + sudo: opts.sudo, detached: true, }); @@ -257,7 +233,7 @@ export async function runPairInSandbox( } }; for (;;) { - await new Promise((resolve) => setTimeout(resolve, 10_000)); + await new Promise((resolve) => setTimeout(resolve, STEP_POLL_INTERVAL_MS)); try { await emitNewOutput(262_144); const status = await sandbox!.runCommand({ @@ -284,6 +260,45 @@ export async function runPairInSandbox( } }; + // -- Bootstrap: what actions/checkout + setup-node + the runner image give + // us for free on GitHub Actions. + await run("install docker", "dnf install -y -q docker", { sudo: true }); + await sandbox.runCommand({ cmd: "dockerd", sudo: true, detached: true }); + await run( + "wait for dockerd", + `for i in $(seq ${DOCKERD_READY_TIMEOUT_SEC}); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1`, + { sudo: true }, + ); + // The harness shells out to `docker` as the unprivileged user; inside a + // single-tenant ephemeral VM, opening the socket is the simple safe way. + await run("open docker socket", "chmod 666 /var/run/docker.sock", { + sudo: true, + }); + + // The agent-skills submodule is public; rewrite its SSH URL to HTTPS since + // the VM has no GitHub SSH identity. The rewrite must live in --global + // config: the child `git clone` a submodule update spawns doesn't read the + // superproject's repo-local config. + await run( + "checkout submodules", + `git config --global url."https://github.com/".insteadOf "git@github.com:"\ngit submodule update --init --recursive`, + ); + + await run("install pnpm", "npm install -g pnpm@10.24.0", { sudo: true }); + await run("pnpm install", "pnpm install --frozen-lockfile"); + + // `pnpm eval` reads the repo-root .env (node --env-file). + await sandbox.writeFiles([ + { + path: ".env", + content: Buffer.from( + Object.entries(options.agentEnv) + .map(([key, value]) => `${key}=${value}`) + .join("\n") + "\n", + ), + }, + ]); + // -- The actual matrix job: same command line as eval-refresh.yml. const evalArgs = [ `--experiment "${pair.experiment}"`, @@ -295,7 +310,7 @@ export async function runPairInSandbox( .filter(Boolean) .join(" "); options.onPhase?.("eval"); - await runPolled("run eval", `pnpm eval -- ${evalArgs}`); + await run("run eval", `pnpm eval -- ${evalArgs}`); options.onPhase?.("collect"); // -- Collect: the artifact-upload equivalent. The results tree (scores, From 038f004e6233b600cc07fdd2b1328afa7604b755 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 00:49:18 +0100 Subject: [PATCH 7/9] =?UTF-8?q?feat:=20warm-boot=20snapshots=20=E2=80=94?= =?UTF-8?q?=20pre-baked=20docker,=20deps,=20and=20stack=20images?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build a Vercel Sandbox snapshot once per input key (lockfile, sandbox Dockerfile, pinned CLI versions; ~3 min): dnf docker, pnpm+node_modules, the agent sandbox base image, and the full Supabase stack pulled via a throwaway supabase start. Pairs boot from it, restart dockerd, fetch the target revision, and run — measured 3.6 min total for a CLI pair that took 7.4 cold (the eval itself halves too: supabase start pulls nothing). Lookup is by builder-sandbox name via Snapshot.list; any failure falls back to cold git-source boots (--no-snapshot forces it). Two traps found live: sandboxes auto-snapshot on stop by default (persistent: false on throwaway VMs — 435 GB had accumulated), and a snapshot inherits the builder's /tmp, so step files carry a per-session nonce lest a poller read a stale exit file and declare an unrun step done. Co-Authored-By: Claude Fable 5 --- packages/vercel-runner/README.md | 24 +- packages/vercel-runner/src/run.ts | 16 ++ packages/vercel-runner/src/sandbox-job.ts | 300 +++++++--------------- packages/vercel-runner/src/snapshot.ts | 228 ++++++++++++++++ packages/vercel-runner/src/vm-steps.ts | 223 ++++++++++++++++ 5 files changed, 571 insertions(+), 220 deletions(-) create mode 100644 packages/vercel-runner/src/snapshot.ts create mode 100644 packages/vercel-runner/src/vm-steps.ts diff --git a/packages/vercel-runner/README.md b/packages/vercel-runner/README.md index 25b9e8dc..0b12946f 100644 --- a/packages/vercel-runner/README.md +++ b/packages/vercel-runner/README.md @@ -94,11 +94,21 @@ checks inside the VM): what the current matrix needs. - **Cost** is metered on *active* CPU (~$0.128/h) + provisioned memory (~$0.021/GB-h); an eval pair (mostly waiting on the model) costs cents. -- **Cold start** is the main overhead: dnf + dockerd + pnpm install + pulling - the Supabase images adds ~4–6 minutes per VM vs. a GitHub runner's warm - image cache. A [sandbox snapshot](https://vercel.com/docs/sandbox/concepts/snapshots) - with docker + node_modules + Supabase images pre-baked would cut this to - seconds and is the obvious next step. +- **Warm-boot snapshots kill the cold start** (`src/snapshot.ts`): docker, + node_modules, the agent sandbox image, and the pulled Supabase stack images + are pre-baked into a [sandbox snapshot](https://vercel.com/docs/sandbox/concepts/snapshots), + built once per input key (lockfile + sandbox Dockerfile + pinned CLI + versions; ~3 min) and resolved by builder-sandbox name. A warm pair boots, + restarts dockerd, fetches the target revision, and is running its eval in + well under a minute — measured 0.8 min *total* for a tools-mode pair that + took ~2 min cold (CLI pairs save more: `supabase start` pulls nothing). + Any snapshot failure falls back to cold git-source boots; `--no-snapshot` + forces that. Expected CI wall-clock: scheduled regression ~10 min; full + 166-pair matrix ≈ creation ramp (~4 min at 46 sandboxes/min, the Pro + 200 vCPUs/min limit) + the slowest eval — ~13–18 min warm. +- **Sandboxes auto-snapshot on stop by default** ("persistent") — throwaway + eval VMs must pass `persistent: false`, or a full-matrix run banks 166 + multi-GB snapshots (435 GB had accumulated before the flag). - **Not built here** (out of spike scope): the durable-queue dispatch the - issue floats, and snapshot caching. Committing/PR-ing the exported JSON - stays in the workflow's existing steps, which are unchanged. + issue floats. Committing/PR-ing the exported JSON stays in the workflow's + existing steps, which are unchanged. diff --git a/packages/vercel-runner/src/run.ts b/packages/vercel-runner/src/run.ts index 320b3699..2941c104 100644 --- a/packages/vercel-runner/src/run.ts +++ b/packages/vercel-runner/src/run.ts @@ -22,6 +22,7 @@ import { type SandboxJobPhase, type SandboxJobResult, } from "./sandbox-job.js"; +import { ensureSnapshot } from "./snapshot.js"; const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); @@ -138,6 +139,20 @@ async function main() { "https://github.com/", ); + // Warm-boot snapshot: docker, node_modules, the agent sandbox image, and + // the Supabase stack images pre-baked, cutting per-pair bootstrap from + // ~5 minutes to well under one. Built once per input key (~10m); any + // failure falls back to cold git-source boots. + const snapshotId = rawArgs.includes("--no-snapshot") + ? undefined + : await ensureSnapshot({ + root: ROOT, + repoUrl, + revision, + githubToken, + vcpus, + }); + // Same fan-out/parallelism model as the Actions matrix: every pair gets its // own isolated machine; only the number in flight at once is capped. const queue: EvalPair[] = [...pairs]; @@ -181,6 +196,7 @@ async function main() { repoUrl, revision, githubToken, + snapshotId, runs, timeoutSec, vcpus, diff --git a/packages/vercel-runner/src/sandbox-job.ts b/packages/vercel-runner/src/sandbox-job.ts index d8058dea..1eb32e49 100644 --- a/packages/vercel-runner/src/sandbox-job.ts +++ b/packages/vercel-runner/src/sandbox-job.ts @@ -9,60 +9,29 @@ * Supabase CLI inside that container spawns the stack as sibling containers on * that same daemon. Nothing in packages/sandbox changes; only the machine the * daemon runs on does. + * + * With a warm-boot snapshot (see snapshot.ts) the VM starts from a filesystem + * that already carries docker, node_modules, and the pulled images; the boot + * then only restarts dockerd and fetches the target revision. */ import { mkdirSync, rmSync, mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { Writable } from "node:stream"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { Sandbox } from "@vercel/sandbox"; import type { EvalPair } from "./discover.js"; +import { + SANDBOX_CWD, + createSandbox, + createStepRunner, + prefixedWriter, + startDocker, +} from "./vm-steps.js"; const execFileAsync = promisify(execFile); -/** Where the git source is checked out inside the sandbox. */ -const SANDBOX_CWD = "/vercel/sandbox"; - -/** Ready-poll budget for dockerd inside the VM. */ -const DOCKERD_READY_TIMEOUT_SEC = 60; - -/** - * Backoff schedule for retrying sandbox creation. A full-matrix fan-out can - * brush the vCPU allocation rate limit or hit a transient API error; like the - * sandbox-image build retry in packages/sandbox, every failure is retried (no - * stable error taxonomy) and a deterministic failure merely wastes the - * schedule before surfacing. - */ -const CREATE_RETRY_DELAYS_MS = [20_000, 60_000]; - -/** Poll cadence for detached step commands (status + incremental log tail). */ -const STEP_POLL_INTERVAL_MS = 5_000; - -/** - * Minimum spacing between sandbox creations across the whole dispatch. Pro - * allocates at most 200 vCPUs/min; at 4 vCPUs per sandbox that's 50 - * creations/min, so ~1.3s spacing (≈46/min) ramps a wide fan-out cleanly - * instead of bouncing off 429s. Concurrency (sandboxes in flight) is - * effectively unlimited by comparison (2,000 on Pro). - */ -const CREATE_SPACING_MS = 1_300; -let nextCreateSlotAt = 0; - -async function waitForCreateSlot(): Promise { - for (;;) { - const now = Date.now(); - if (now >= nextCreateSlotAt) { - // Synchronous check-and-claim: no await between read and write, so - // concurrent jobs on the single event loop can't grab the same slot. - nextCreateSlotAt = now + CREATE_SPACING_MS; - return; - } - await new Promise((resolve) => setTimeout(resolve, nextCreateSlotAt - now)); - } -} - /** Coarse lifecycle phase of a pair's sandbox, for fleet progress reporting. */ export type SandboxJobPhase = "create" | "bootstrap" | "eval" | "collect"; @@ -76,6 +45,8 @@ export interface SandboxJobOptions { revision: string; /** Token able to read the repo (it's internal) — goes into the git source. */ githubToken: string; + /** Warm-boot snapshot to start from; omitted means a cold git-source boot. */ + snapshotId?: string; runs: number; timeoutSec: number; vcpus: number; @@ -87,24 +58,6 @@ export interface SandboxJobOptions { resultsDir: string; } -/** - * Explicit Vercel API credentials, when the environment carries them. The SDK - * does NOT read VERCEL_TOKEN/VERCEL_TEAM_ID/VERCEL_PROJECT_ID on its own — its - * only fallbacks are $VERCEL_OIDC_TOKEN and interactively cached dev - * credentials (`vercel link`), and CI has neither. - */ -function credentialsFromEnv(): - | { token: string; teamId: string; projectId: string } - | undefined { - const { VERCEL_TOKEN, VERCEL_TEAM_ID, VERCEL_PROJECT_ID } = process.env; - if (!VERCEL_TOKEN || !VERCEL_TEAM_ID || !VERCEL_PROJECT_ID) return undefined; - return { - token: VERCEL_TOKEN, - teamId: VERCEL_TEAM_ID, - projectId: VERCEL_PROJECT_ID, - }; -} - export interface SandboxJobResult { pair: EvalPair; ok: boolean; @@ -116,24 +69,6 @@ export interface SandboxJobResult { sandboxId?: string; } -/** Prefix every output line so interleaved parallel jobs stay readable. */ -function prefixedWriter(prefix: string, target: NodeJS.WriteStream): Writable { - let pending = ""; - return new Writable({ - write(chunk, _encoding, callback) { - pending += chunk.toString(); - const lines = pending.split("\n"); - pending = lines.pop() ?? ""; - for (const line of lines) target.write(`${prefix} ${line}\n`); - callback(); - }, - final(callback) { - if (pending) target.write(`${prefix} ${pending}\n`); - callback(); - }, - }); -} - export async function runPairInSandbox( options: SandboxJobOptions, ): Promise { @@ -146,146 +81,85 @@ export async function runPairInSandbox( let sandbox: Sandbox | undefined; try { - log(`creating sandbox (vcpus=${options.vcpus}, timeout=${Math.round(options.sandboxTimeoutMs / 60000)}m, rev=${options.revision.slice(0, 8)})`); + log( + `creating sandbox (vcpus=${options.vcpus}, timeout=${Math.round(options.sandboxTimeoutMs / 60000)}m, ` + + `rev=${options.revision.slice(0, 8)}${options.snapshotId ? ", warm boot" : ", cold boot"})`, + ); options.onPhase?.("create"); - for (let attempt = 0; ; attempt++) { - try { - await waitForCreateSlot(); - sandbox = await Sandbox.create({ - ...credentialsFromEnv(), - runtime: "node22", - resources: { vcpus: options.vcpus }, - timeout: options.sandboxTimeoutMs, - // Makes the Vercel dashboard's Sandboxes view a live per-CI-run - // board, filterable by these keys (max 5 tags). - tags: { - runner: "supabase-evals", - run: process.env.GITHUB_RUN_ID ?? "local", - experiment: pair.experiment, - eval: pair.evalId, - }, - source: { - type: "git", - url: options.repoUrl, - revision: options.revision, - // Vercel clones with these as basic-auth; x-access-token is - // GitHub's conventional username for token auth. - username: "x-access-token", - password: options.githubToken, - }, - }); - break; - } catch (err) { - const delayMs = CREATE_RETRY_DELAYS_MS[attempt]; - if (delayMs === undefined) throw err; - stderr.write( - `sandbox creation failed (attempt ${attempt + 1}/${CREATE_RETRY_DELAYS_MS.length + 1}), ` + - `retrying in ${delayMs / 1000}s: ${err instanceof Error ? err.message : err}\n`, - ); - await new Promise((resolve) => setTimeout(resolve, delayMs)); - } - } + sandbox = await createSandbox( + { + resources: { vcpus: options.vcpus }, + timeout: options.sandboxTimeoutMs, + // Sandboxes auto-snapshot on stop by default ("persistent"); a + // throwaway eval VM must not — a full-matrix run would otherwise + // bank 166 multi-GB snapshots (it did: 435 GB accumulated before + // this flag). + persistent: false, + // Makes the Vercel dashboard's Sandboxes view a live per-CI-run + // board, filterable by these keys (max 5 tags). + tags: { + runner: "supabase-evals", + run: process.env.GITHUB_RUN_ID ?? "local", + experiment: pair.experiment, + eval: pair.evalId, + }, + // A snapshot source carries its runtime; git sources need one. + ...(options.snapshotId + ? { source: { type: "snapshot" as const, snapshotId: options.snapshotId } } + : { + runtime: "node22", + source: { + type: "git" as const, + url: options.repoUrl, + revision: options.revision, + // Vercel clones with these as basic-auth; x-access-token is + // GitHub's conventional username for token auth. + username: "x-access-token", + password: options.githubToken, + }, + }), + }, + stderr, + ); log(`sandbox ${sandbox.name} created`); options.onPhase?.("bootstrap"); - // Every step runs decoupled from any live log stream: launch detached - // with output going to a file in the VM, then poll with short commands - // (status sentinel + incremental tail). A command held on one streaming - // connection dies with "Stream ended before command finished" when that - // connection drops even though the command and the VM are fine — at - // full-matrix scale (~1,000 step commands per run) that's a certainty, - // and it killed bootstrap steps in practice, not just the long eval. - let stepCounter = 0; - const run = async ( - step: string, - cmd: string, - opts: { sudo?: boolean } = {}, - ) => { - log(`--- ${step}`); - const id = ++stepCounter; - const logPath = `/tmp/step-${id}.log`; - const exitPath = `/tmp/step-${id}.exit`; - await sandbox!.runCommand({ - cmd: "bash", - args: [ - "-c", - `(set -euo pipefail\n${cmd}\n) >${logPath} 2>&1; echo $? >${exitPath}`, - ], - cwd: SANDBOX_CWD, - sudo: opts.sudo, - detached: true, - }); + const run = createStepRunner(sandbox, { stdout, stderr, log }); - let offset = 0; - let pollFailures = 0; - const emitNewOutput = async (limit?: number) => { - const chunk = await sandbox!.runCommand({ - cmd: "bash", - args: [ - "-c", - `tail -c +${offset + 1} ${logPath}${limit ? ` | head -c ${limit}` : ""}`, - ], - }); - const data = await chunk.stdout(); - if (data) { - offset += Buffer.byteLength(data); - stdout.write(data); - } - }; - for (;;) { - await new Promise((resolve) => setTimeout(resolve, STEP_POLL_INTERVAL_MS)); - try { - await emitNewOutput(262_144); - const status = await sandbox!.runCommand({ - cmd: "bash", - args: ["-c", `[ -f ${exitPath} ] && cat ${exitPath} || echo RUNNING`], - }); - const text = (await status.stdout()).trim(); - pollFailures = 0; - if (text === "RUNNING") continue; - // Output written between the last chunk and completion is still in - // the file; drain it before reporting the outcome. - await emitNewOutput(); - if (text === "0") return; - throw new Error(`step "${step}" exited with code ${text}`); - } catch (err) { - if (err instanceof Error && err.message.startsWith(`step "`)) throw err; - // Transient control-plane hiccups shouldn't kill a healthy run. - pollFailures += 1; - if (pollFailures >= 6) throw err; - stderr.write( - `poll failed (${pollFailures}/6), retrying: ${err instanceof Error ? err.message : err}\n`, - ); - } - } - }; - - // -- Bootstrap: what actions/checkout + setup-node + the runner image give - // us for free on GitHub Actions. - await run("install docker", "dnf install -y -q docker", { sudo: true }); - await sandbox.runCommand({ cmd: "dockerd", sudo: true, detached: true }); - await run( - "wait for dockerd", - `for i in $(seq ${DOCKERD_READY_TIMEOUT_SEC}); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1`, - { sudo: true }, - ); - // The harness shells out to `docker` as the unprivileged user; inside a - // single-tenant ephemeral VM, opening the socket is the simple safe way. - await run("open docker socket", "chmod 666 /var/run/docker.sock", { - sudo: true, - }); - - // The agent-skills submodule is public; rewrite its SSH URL to HTTPS since - // the VM has no GitHub SSH identity. The rewrite must live in --global - // config: the child `git clone` a submodule update spawns doesn't read the - // superproject's repo-local config. - await run( - "checkout submodules", - `git config --global url."https://github.com/".insteadOf "git@github.com:"\ngit submodule update --init --recursive`, - ); - - await run("install pnpm", "npm install -g pnpm@10.24.0", { sudo: true }); - await run("pnpm install", "pnpm install --frozen-lockfile"); + if (options.snapshotId) { + // Warm boot: the snapshot filesystem already has docker + images + + // node_modules; restart the daemon (processes aren't snapshotted) and + // move the checkout to the target revision. + await startDocker(sandbox, run, { install: false }); + await run( + "checkout revision", + // The token flows through the credential helper, never a command + // line, so it can't leak into step logs on error. + `git config --global credential.helper store\n` + + `printf 'https://x-access-token:%s@github.com\\n' "$GITHUB_TOKEN" > ~/.git-credentials\n` + + `git fetch --quiet origin "$REVISION"\n` + + `git checkout --quiet --force "$REVISION"\n` + + `git submodule update --init --recursive`, + { env: { GITHUB_TOKEN: options.githubToken, REVISION: options.revision } }, + ); + // Reconciles any dependency drift between the snapshot's lockfile and + // the target revision's; a no-op seconds when they match. + await run("pnpm install", "pnpm install --frozen-lockfile"); + } else { + // Cold boot: what actions/checkout + setup-node + the runner image give + // us for free on GitHub Actions. + await startDocker(sandbox, run, { install: true }); + // The agent-skills submodule is public; rewrite its SSH URL to HTTPS + // since the VM has no GitHub SSH identity. The rewrite must live in + // --global config: the child `git clone` a submodule update spawns + // doesn't read the superproject's repo-local config. + await run( + "checkout submodules", + `git config --global url."https://github.com/".insteadOf "git@github.com:"\ngit submodule update --init --recursive`, + ); + await run("install pnpm", "npm install -g pnpm@10.24.0", { sudo: true }); + await run("pnpm install", "pnpm install --frozen-lockfile"); + } // `pnpm eval` reads the repo-root .env (node --env-file). await sandbox.writeFiles([ diff --git a/packages/vercel-runner/src/snapshot.ts b/packages/vercel-runner/src/snapshot.ts new file mode 100644 index 00000000..23e676fa --- /dev/null +++ b/packages/vercel-runner/src/snapshot.ts @@ -0,0 +1,228 @@ +/** + * Warm-boot snapshots: pre-bake everything a pair's VM spends its first ~5 + * minutes on — dnf docker, pnpm + node_modules, the agent sandbox base image, + * and the pulled Supabase stack images — into a Vercel Sandbox snapshot, so + * per-pair sandboxes boot from it and only need a dockerd restart plus a git + * fetch of the target revision. + * + * Snapshots are keyed by their inputs: the builder sandbox is named + * `evals-snap-` and `Snapshot.list({ name })` resolves key → snapshot, + * so a stale key simply misses and triggers a rebuild. Expiry is Vercel's + * default (30 days after last use), which self-renews under regular runs. + */ + +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { Sandbox, Snapshot } from "@vercel/sandbox"; +import { + createSandbox, + createStepRunner, + credentialsFromEnv, + prefixedWriter, + startDocker, +} from "./vm-steps.js"; + +/** Bump to invalidate existing snapshots when the builder steps change. */ +const BUILDER_VERSION = "1"; + +/** Budget for the one-time builder sandbox (image pulls dominate). */ +const BUILDER_TIMEOUT_MS = 30 * 60_000; + +function readVersionConstant(root: string, file: string, name: string): string { + const source = readFileSync(join(root, file), "utf8"); + const match = source.match(new RegExp(`${name} = "([^"]+)"`)); + if (!match) throw new Error(`could not read ${name} from ${file}`); + return match[1]; +} + +/** + * Everything that changes what the snapshot contains: dependency tree, the + * agent sandbox image definition and its pinned versions, and the builder + * itself. Computed from the local working tree — the same inputs the run's + * revision will carry in the common case, and a per-run `pnpm install` + * reconciles any drift. + */ +export function computeSnapshotKey(root: string): string { + const cliVersion = readVersionConstant( + root, + "packages/sandbox/src/supabase.ts", + "SUPABASE_CLI_VERSION", + ); + const skillsVersion = readVersionConstant( + root, + "packages/sandbox/src/skills.ts", + "SKILLS_CLI_VERSION", + ); + return createHash("sha256") + .update(`builder:${BUILDER_VERSION}\n`) + .update(`cli:${cliVersion}\n`) + .update(`skills:${skillsVersion}\n`) + .update(readFileSync(join(root, "packages/sandbox/Dockerfile"))) + .update(readFileSync(join(root, "pnpm-lock.yaml"))) + .digest("hex") + .slice(0, 12); +} + +export interface EnsureSnapshotOptions { + root: string; + repoUrl: string; + revision: string; + githubToken: string; + vcpus: number; +} + +/** + * Resolve (or build) the warm-boot snapshot for the current key. Returns + * undefined on any failure so the caller falls back to cold git-source boots + * — the snapshot is an optimization, never a requirement. + */ +export async function ensureSnapshot( + options: EnsureSnapshotOptions, +): Promise { + const stdout = prefixedWriter("[snapshot]", process.stdout); + const stderr = prefixedWriter("[snapshot]", process.stderr); + const log = (message: string) => stdout.write(`${message}\n`); + + try { + const key = computeSnapshotKey(options.root); + const name = `evals-snap-${key}`; + + const existing = await findSnapshot(name); + if (existing) { + log(`reusing snapshot ${existing} (key ${key})`); + return existing; + } + + log(`no snapshot for key ${key} — building one (~10m, once per key)`); + const start = Date.now(); + const snapshotId = await buildSnapshot(name, options, { stdout, stderr, log }); + log( + `snapshot ${snapshotId} built in ${Math.round((Date.now() - start) / 60000)}m`, + ); + return snapshotId; + } catch (err) { + stderr.write( + `snapshot unavailable, falling back to cold boots: ${err instanceof Error ? err.message : err}\n`, + ); + return undefined; + } finally { + stdout.end(); + stderr.end(); + } +} + +async function findSnapshot(name: string): Promise { + const result = await Snapshot.list({ name, ...credentialsFromEnv() }); + let newest: { id: string; createdAt: number } | undefined; + for await (const snapshot of result) { + if (snapshot.status !== "created") continue; + if (!newest || snapshot.createdAt > newest.createdAt) { + newest = { id: snapshot.id, createdAt: snapshot.createdAt }; + } + } + return newest?.id; +} + +async function buildSnapshot( + name: string, + options: EnsureSnapshotOptions, + io: { + stdout: ReturnType; + stderr: ReturnType; + log: (message: string) => void; + }, +): Promise { + const cliVersion = readVersionConstant( + options.root, + "packages/sandbox/src/supabase.ts", + "SUPABASE_CLI_VERSION", + ); + const skillsVersion = readVersionConstant( + options.root, + "packages/sandbox/src/skills.ts", + "SKILLS_CLI_VERSION", + ); + + const sandbox = await createSandbox( + { + name, + runtime: "node22", + resources: { vcpus: options.vcpus }, + timeout: BUILDER_TIMEOUT_MS, + // Rebuilds evict the previous key-less snapshot of the same builder + // name; the snapshot itself never expires while in regular use. + keepLastSnapshots: { count: 1 }, + tags: { + runner: "supabase-evals", + purpose: "snapshot-builder", + run: process.env.GITHUB_RUN_ID ?? "local", + }, + source: { + type: "git", + url: options.repoUrl, + revision: options.revision, + username: "x-access-token", + password: options.githubToken, + }, + }, + io.stderr, + ); + io.log(`builder sandbox ${sandbox.name} created`); + try { + const run = createStepRunner(sandbox, io); + await startDocker(sandbox, run, { install: true }); + await run( + "checkout submodules", + `git config --global url."https://github.com/".insteadOf "git@github.com:"\ngit submodule update --init --recursive`, + ); + await run("install pnpm", "npm install -g pnpm@10.24.0", { sudo: true }); + await run("pnpm install", "pnpm install --frozen-lockfile"); + + // The exact image ensureSupabaseSandboxImage builds per session — with + // the tag already in the daemon, the eval-time build is a cache hit. + await run( + "build agent sandbox image", + `docker build --build-arg SKILLS_CLI_VERSION=${skillsVersion} ` + + `--tag supabase-evals-sandbox:base-skills-${skillsVersion} - < packages/sandbox/Dockerfile`, + ); + + // Pre-pull the Supabase stack by running a throwaway project once with + // the pinned CLI: `supabase start` pulls every service image into + // /var/lib/docker (snapshotted), then the project is torn down so no + // containers or volumes leak into the snapshot. + // The VM is Amazon Linux (rpm), unlike the Debian agent container which + // installs the .deb — same pinned version, so the same image tags get + // pulled. + await run( + "install supabase CLI", + `ARCH="$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')" && ` + + `curl -fsSL "https://github.com/supabase/cli/releases/download/v${cliVersion}/supabase_${cliVersion}_linux_$ARCH.rpm" -o /tmp/supabase.rpm && ` + + `rpm -i /tmp/supabase.rpm && rm /tmp/supabase.rpm`, + { sudo: true }, + ); + await run( + "prewarm supabase images", + `mkdir -p /tmp/prewarm && cd /tmp/prewarm && supabase init && supabase start && supabase stop --no-backup && cd / && rm -rf /tmp/prewarm`, + ); + + // Hygiene: nothing secret or run-specific may live in the snapshot. + await run( + "scrub credentials", + `git config --global --remove-section credential 2>/dev/null || true\nrm -f ~/.git-credentials .env`, + ); + // Step/state files from this build must not leak into warm boots (the + // nonce in step paths already prevents collisions; this is hygiene). + await run("scrub temp files", "rm -f /tmp/step-* /tmp/eval-results.tgz", { + sudo: true, + }); + + io.log("snapshotting (stops the builder)…"); + const snapshot = await sandbox.snapshot(); + return snapshot.snapshotId; + } catch (err) { + // snapshot() stops the sandbox on success; on failure stop it ourselves. + await sandbox.stop().catch(() => undefined); + throw err; + } +} diff --git a/packages/vercel-runner/src/vm-steps.ts b/packages/vercel-runner/src/vm-steps.ts new file mode 100644 index 00000000..4ad4d63d --- /dev/null +++ b/packages/vercel-runner/src/vm-steps.ts @@ -0,0 +1,223 @@ +/** + * Shared microVM plumbing for the per-pair eval job and the snapshot builder: + * prefixed log writers, rate-shaped sandbox creation, and the polled step + * runner every VM command goes through. + */ + +import { Writable } from "node:stream"; +import { Sandbox } from "@vercel/sandbox"; + +/** Where the git source is checked out inside the sandbox. */ +export const SANDBOX_CWD = "/vercel/sandbox"; + +/** Ready-poll budget for dockerd inside the VM. */ +const DOCKERD_READY_TIMEOUT_SEC = 60; + +/** + * Backoff schedule for retrying sandbox creation. A full-matrix fan-out can + * brush the vCPU allocation rate limit or hit a transient API error; like the + * sandbox-image build retry in packages/sandbox, every failure is retried (no + * stable error taxonomy) and a deterministic failure merely wastes the + * schedule before surfacing. + */ +const CREATE_RETRY_DELAYS_MS = [20_000, 60_000]; + +/** Poll cadence for detached step commands (status + incremental log tail). */ +const STEP_POLL_INTERVAL_MS = 5_000; + +/** + * Minimum spacing between sandbox creations across the whole dispatch. Pro + * allocates at most 200 vCPUs/min; at 4 vCPUs per sandbox that's 50 + * creations/min, so ~1.3s spacing (≈46/min) ramps a wide fan-out cleanly + * instead of bouncing off 429s. Concurrency (sandboxes in flight) is + * effectively unlimited by comparison (2,000 on Pro). + */ +const CREATE_SPACING_MS = 1_300; +let nextCreateSlotAt = 0; + +async function waitForCreateSlot(): Promise { + for (;;) { + const now = Date.now(); + if (now >= nextCreateSlotAt) { + // Synchronous check-and-claim: no await between read and write, so + // concurrent jobs on the single event loop can't grab the same slot. + nextCreateSlotAt = now + CREATE_SPACING_MS; + return; + } + await new Promise((resolve) => setTimeout(resolve, nextCreateSlotAt - now)); + } +} + +/** + * Explicit Vercel API credentials, when the environment carries them. The SDK + * does NOT read VERCEL_TOKEN/VERCEL_TEAM_ID/VERCEL_PROJECT_ID on its own — its + * only fallbacks are $VERCEL_OIDC_TOKEN and interactively cached dev + * credentials (`vercel link`), and CI has neither. + */ +export function credentialsFromEnv(): + | { token: string; teamId: string; projectId: string } + | undefined { + const { VERCEL_TOKEN, VERCEL_TEAM_ID, VERCEL_PROJECT_ID } = process.env; + if (!VERCEL_TOKEN || !VERCEL_TEAM_ID || !VERCEL_PROJECT_ID) return undefined; + return { + token: VERCEL_TOKEN, + teamId: VERCEL_TEAM_ID, + projectId: VERCEL_PROJECT_ID, + }; +} + +/** Prefix every output line so interleaved parallel jobs stay readable. */ +export function prefixedWriter( + prefix: string, + target: NodeJS.WriteStream, +): Writable { + let pending = ""; + return new Writable({ + write(chunk, _encoding, callback) { + pending += chunk.toString(); + const lines = pending.split("\n"); + pending = lines.pop() ?? ""; + for (const line of lines) target.write(`${prefix} ${line}\n`); + callback(); + }, + final(callback) { + if (pending) target.write(`${prefix} ${pending}\n`); + callback(); + }, + }); +} + +/** Rate-shaped, retried Sandbox.create. */ +export async function createSandbox( + params: Parameters[0], + stderr: Writable, +): Promise { + for (let attempt = 0; ; attempt++) { + try { + await waitForCreateSlot(); + return await Sandbox.create({ ...credentialsFromEnv(), ...params }); + } catch (err) { + const delayMs = CREATE_RETRY_DELAYS_MS[attempt]; + if (delayMs === undefined) throw err; + stderr.write( + `sandbox creation failed (attempt ${attempt + 1}/${CREATE_RETRY_DELAYS_MS.length + 1}), ` + + `retrying in ${delayMs / 1000}s: ${err instanceof Error ? err.message : err}\n`, + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } +} + +export type StepRunner = ( + step: string, + cmd: string, + opts?: { sudo?: boolean; env?: Record }, +) => Promise; + +/** + * Build the polled step runner for a sandbox. Every step runs decoupled from + * any live log stream: launch detached with output going to a file in the VM, + * then poll with short commands (status sentinel + incremental tail). A + * command held on one streaming connection dies with "Stream ended before + * command finished" when that connection drops even though the command and + * the VM are fine — at full-matrix scale (~1,000 step commands per run) + * that's a certainty, and it killed bootstrap steps in practice, not just the + * long eval. + */ +export function createStepRunner( + sandbox: Sandbox, + io: { stdout: Writable; stderr: Writable; log: (message: string) => void }, +): StepRunner { + // Step files carry a per-session nonce: a sandbox booted from a snapshot + // inherits the builder's /tmp — a poller reading a stale exit file from a + // colliding path would declare the step done while the command still runs + // (it did: "run eval" once "passed" in 5 seconds with no eval run). + const nonce = Math.random().toString(36).slice(2, 10); + let stepCounter = 0; + return async (step, cmd, opts = {}) => { + io.log(`--- ${step}`); + const id = ++stepCounter; + const logPath = `/tmp/step-${nonce}-${id}.log`; + const exitPath = `/tmp/step-${nonce}-${id}.exit`; + await sandbox.runCommand({ + cmd: "bash", + args: [ + "-c", + `rm -f ${logPath} ${exitPath}; (set -euo pipefail\n${cmd}\n) >${logPath} 2>&1; echo $? >${exitPath}`, + ], + cwd: SANDBOX_CWD, + sudo: opts.sudo, + env: opts.env, + detached: true, + }); + + let offset = 0; + let pollFailures = 0; + const emitNewOutput = async (limit?: number) => { + const chunk = await sandbox.runCommand({ + cmd: "bash", + args: [ + "-c", + `tail -c +${offset + 1} ${logPath}${limit ? ` | head -c ${limit}` : ""}`, + ], + }); + const data = await chunk.stdout(); + if (data) { + offset += Buffer.byteLength(data); + io.stdout.write(data); + } + }; + for (;;) { + await new Promise((resolve) => setTimeout(resolve, STEP_POLL_INTERVAL_MS)); + try { + await emitNewOutput(262_144); + const status = await sandbox.runCommand({ + cmd: "bash", + args: ["-c", `[ -f ${exitPath} ] && cat ${exitPath} || echo RUNNING`], + }); + const text = (await status.stdout()).trim(); + pollFailures = 0; + if (text === "RUNNING") continue; + // Output written between the last chunk and completion is still in + // the file; drain it before reporting the outcome. + await emitNewOutput(); + if (text === "0") return; + throw new Error(`step "${step}" exited with code ${text}`); + } catch (err) { + if (err instanceof Error && err.message.startsWith(`step "`)) throw err; + // Transient control-plane hiccups shouldn't kill a healthy run. + pollFailures += 1; + if (pollFailures >= 6) throw err; + io.stderr.write( + `poll failed (${pollFailures}/6), retrying: ${err instanceof Error ? err.message : err}\n`, + ); + } + } + }; +} + +/** + * Start dockerd and open its socket to the unprivileged user (inside a + * single-tenant ephemeral VM, opening the socket is the simple safe way). + * `install` additionally dnf-installs docker — snapshot boots skip it, since + * the package (and the pulled images) are already on the snapshot filesystem; + * only the daemon process needs restarting (processes aren't snapshotted). + */ +export async function startDocker( + sandbox: Sandbox, + run: StepRunner, + options: { install: boolean }, +): Promise { + if (options.install) { + await run("install docker", "dnf install -y -q docker", { sudo: true }); + } + await sandbox.runCommand({ cmd: "dockerd", sudo: true, detached: true }); + await run( + "wait for dockerd", + `for i in $(seq ${DOCKERD_READY_TIMEOUT_SEC}); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1`, + { sudo: true }, + ); + await run("open docker socket", "chmod 666 /var/run/docker.sock", { + sudo: true, + }); +} From 862e9f94f6fa13a56774b2e104a3e62664c8a57f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:18:17 +0000 Subject: [PATCH 8/9] chore: refresh eval results --- apps/web/src/data/eval-results.json | 14344 +++++++--------- .../web/src/data/regression-eval-results.json | 567 +- 2 files changed, 6582 insertions(+), 8329 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 556790c2..9866beed 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -160,12 +160,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "queue depth 1 -> 2" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "notes": "function removed the seeded message (id 9) from the queue" } ], "skills": { @@ -181,59 +181,51 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job invoke edge function net.http_post queue\", limit: 6) { nodes { title href content } } }", + "query": "{ q1: searchDocs(query: \"Supabase Queues create queue send read pop pgmq_public API\", limit: 5) { nodes { title href content } } q2: searchDocs(query: \"Supabase Cron schedule job every minute pg_cron enqueue queue\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/webhook-debugging-guide-M8sk47", - "title": "Webhook debugging guide" + "url": "https://supabase.com/docs/guides/database/extensions/pgmq", + "title": "pgmq: Queues" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - } - ], - "resultChars": 62880 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"queues read pop delete messages consume edge function pgmq_public rpc\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" } ], - "resultChars": 60050 + "resultChars": 65771 } ] }, @@ -393,7 +385,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -403,42 +395,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"d6d5fe0c-99bd-4131-bf20-272b4ea4e9fd\",\"metric\":\"steps_a_mro0usf9\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"23a37d85-294e-4a36-8805-d4390af1a8ae\",\"metric\":\"steps_a_mry6y5fi\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"d6d5fe0c-99bd-4131-bf20-272b4ea4e9fd\",\"metric\":\"steps_a_mro0usf9\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"23a37d85-294e-4a36-8805-d4390af1a8ae\",\"metric\":\"steps_a_mry6y5fi\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"dca2d817-6fa6-47ce-9655-c733eb8a7c63\",\"metric\":\"steps_b_mro0usf9\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"295160e9-46f7-4fb8-99a7-c36700a53f37\",\"metric\":\"steps_b_mry6y5fi\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -453,67 +445,59 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function service role key apikey header authenticate user getUser\", limit: 5) { nodes { title href content } } }", + "source": "web_fetch", + "query": "List any breaking changes or notable updates related to edge functions, API keys (publishable/secret/anon/service_role), environment variables exposed to edge functions, and the supabase-js server client. Include dates.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 34937 + "resultChars": 1872 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"withSupabase @supabase/server edge function auth publishable secret authMode ctx supabaseAdmin getUser\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function verify_jwt authorization service role dual auth\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" } ], - "resultChars": 46083 + "resultChars": 44094 + }, + { + "source": "web_search", + "query": "supabase functions serve \"failed to determine entrypoint\" main worker boot error edge-runtime", + "pages": [], + "resultChars": 1907 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-4.8/build-functions-005-dual-auth-user-secret.json" }, { @@ -554,7 +538,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019f6c9d-f613-743e-92ba-d858089774ab/receipt-alpha.pdf, 019f6c9d-f613-743e-92ba-d858089774ab/receipt-beta.pdf" + "notes": "saw: 019f9165-e3f9-75c8-8748-c219d6232564/receipt-alpha.pdf, 019f9165-e3f9-75c8-8748-c219d6232564/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -575,7 +559,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates private user-files bucket, owner-scoped SELECT and INSERT policies on storage.objects for authenticated users using first folder = auth.uid(), keeps RLS enabled (does not disable it), and provides supabase-js createSignedUrl code with expiry. No public bucket/getPublicUrl/service-role misuse." + "judgeNotes": "Meets all requirements: private bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, RLS kept enabled/verified, and uses createSignedUrl with an expiry for temporary sharing." } ], "skills": { @@ -591,44 +575,45 @@ "calls": [ { "source": "search_docs", - "query": "{ storage: searchDocs(query: \"storage bucket RLS policy user folder path auth.uid\", limit: 5) { nodes { title href content } } signed: searchDocs(query: \"createSignedUrl temporary link expires storage\", limit: 4) { nodes { title href content } } }", + "query": "{\n storage: searchDocs(query: \"storage bucket RLS policy own folder user id first path segment\", limit: 5) {\n nodes { title href content }\n }\n signed: searchDocs(query: \"createSignedUrl temporary link expires supabase-js download\", limit: 4) {\n nodes { title href content }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" }, { "url": "https://supabase.com/docs/guides/storage/serving/downloads", "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" } ], - "resultChars": 68372 + "resultChars": 47304 } ] }, @@ -667,12 +652,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "3 passed, 2 failed" + "notes": "5 passed, 5 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "Correctly identifies `posts` as having a broken tenant isolation SELECT policy, grounded in pgTAP failure showing org1 member can read org2 posts. Does not blame `notes` and treats test results as authoritative." + "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members of any org can read posts from other orgs due to a missing `m.org_id = posts.org_id` predicate, and grounds this in pgTAP failures. It also correctly notes `notes` passes isolation tests rather than blaming it. Extra discussion of `memberships` does not undermine the required conclusion." } ], "skills": { @@ -760,37 +745,39 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search gte-small embeddings match_document_sections RLS\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"semantic search gte-small embeddings dimensions match_documents rpc\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"semantic search gte-small embeddings dimensions match_document_sections rpc RLS\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, { "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", "title": "Choosing your Compute Add-on" }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, { "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" } ], - "resultChars": 57915 + "resultChars": 53298 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-opus-4.8/build-vectors-001-rag-with-permissions.json" }, { @@ -820,12 +807,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets requirements: preserves app scrape, adds Supabase HTTPS scrape with correct metrics_path, Basic Auth using password_file, project target on .supabase.co:443, and docker-compose mounts the secrets directory containing the password file read-only." + "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, preserves the app job, and docker-compose mounts the secrets directory containing the password file." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes concrete live setup steps: replace project ref, create a Supabase Secret API key, write it to the expected secret file mounted by Compose, and reload/start the Compose stack. It also provides verification via curl, Prometheus targets, and PromQL/Grafana guidance. No hardcoded real secret or mismatched setup detected." + "judgeNotes": "README includes Secret API key creation, matching secret file placement, project ref replacement, Compose start/reload steps, and concrete verification via Prometheus targets." } ], "skills": { @@ -839,14 +826,25 @@ }, "docs": { "calls": [ + { + "source": "web_fetch", + "query": "What is the exact Prometheus metrics endpoint URL for a Supabase project, what authentication does it require (username/password), what plan is required, and what is an example Prometheus scrape_config for it? Include any notes about scrape interval, basic_auth, and the metrics-only service_role.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/telemetry/metrics.md" + } + ], + "resultChars": 1062 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint observability Grafana integration\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"prometheus metrics endpoint scrape config observability\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", @@ -857,32 +855,17 @@ "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - } - ], - "resultChars": 24393 - }, - { - "source": "web_fetch", - "query": "What is the exact Supabase project metrics endpoint URL, what authentication does it use (username/password), what is the recommended Prometheus scrape config (job, scrape_interval, metrics_path, basic_auth, scheme), and any Grafana dashboard details? Quote exact config snippets.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics.md" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" } ], - "resultChars": 1190 + "resultChars": 23542 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-4.8/deploy-database-001-prometheus-metrics.json" }, { @@ -935,7 +918,36 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function environment variables secrets Deno.env.get deploy\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 62245 + } + ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -995,25 +1007,25 @@ "calls": [ { "source": "web_fetch", - "query": "List the exact steps to self-host Supabase with Docker. Include: which files/directories to copy (docker-compose.yml, .env.example, volumes/), the exact commands to obtain them, the full list of secrets/env vars that must be set in .env (POSTGRES_PASSWORD, JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, DASHBOARD_USERNAME, DASHBOARD_PASSWORD, SECRET_KEY_BASE, VAULT_ENC_KEY, pooler tenant/keys, etc.), how to generate JWT anon and service_role keys, and any security notes about changing default credentials.", + "query": "Extract the full step-by-step instructions for self-hosting Supabase with Docker. Specifically: which files/directories are needed (docker-compose.yml, .env, volumes), the exact secrets/env vars that must be set and secured (POSTGRES_PASSWORD, JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, DASHBOARD_USERNAME, DASHBOARD_PASSWORD, SECRET_KEY_BASE, VAULT_ENC_KEY, POOLER_TENANT_ID, etc.), how ANON_KEY and SERVICE_ROLE_KEY relate to JWT_SECRET, and the recommended way to generate secure secrets. List every \"Securing your services\" instruction verbatim.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/docker" } ], - "resultChars": 4558 + "resultChars": 5697 }, { "source": "web_fetch", - "query": "List any recent breaking-change entries related to self-hosting, Docker, docker-compose, env vars, JWT keys, API keys (anon/service_role/publishable/secret), or the analytics/logflare/vector/pooler services.", + "query": "List any entries tagged breaking-change or related to self-hosting, docker, docker-compose, JWT keys, API keys (anon/service_role/publishable/secret), or auth key changes. Include dates.", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1453 + "resultChars": 4752 } ] }, @@ -1073,7 +1085,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Identifies soft-delete-only cause and lack of session revocation; implements meaningful revocation by banning auth user and deleting sessions/refresh tokens plus RLS enforcement of deleted flag. Explains JWTs remain valid until expiry for local validation, while DB path is closed immediately due to RLS checking active account, which is consistent with the implemented fix. Correctly distinguishes publishable/frontend/RLS-enforced keys from secret/server-only/RLS-bypassing keys." + "judgeNotes": "Diagnoses soft-delete-only flow, implements session/refresh revocation plus ban and RLS active-account enforcement, explains JWT expiry caveat consistently with the RLS fix, and correctly distinguishes publishable frontend/RLS-enforced keys from secret server-only/RLS-bypassing keys." } ], "skills": { @@ -1086,7 +1098,19 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "What are the new publishable and secret API keys? Which one goes in frontend/browser code and which is backend-only? How does each interact with RLS and which Postgres roles (anon, authenticated, service_role) do they map to? Are they JWTs or opaque tokens, and can they be rotated/revoked? What are their prefixes (sb_publishable_, sb_secret_)?", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/api-keys" + } + ], + "resultChars": 1351 + } + ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", @@ -1139,7 +1163,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly diagnosed that the channel can be SUBSCRIBED while orders INSERT events are silent because public.orders was missing from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified publication membership, and did not alter RLS, policies, courier_locations, or client code." + "judgeNotes": "The assistant correctly identified that the channel subscribed but no INSERT events arrived because orders was missing from the supabase_realtime publication, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified both orders and courier_locations remained in the publication, and did not weaken RLS or policies." } ], "skills": { @@ -1183,17 +1207,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and explicitly listed the recurring 503 gateway failures across the morning of 2026-04-28, covering all 8 failures from 07:00Z–12:00Z." + "judgeNotes": "Identified `image-transform` as the affected function and described the recurring HTTP 503 gateway failures across the morning of 2026-04-28, covering the spread of failures from about 07:00Z to 12:00Z and distinguishing them from the older billing-webhook 503s." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/platform layer before function execution, grounded in the mismatch between API/gateway 503s and clean edge-function runtime 200 logs, and distinguishes them from avatar-upload's function-level 500." + "judgeNotes": "The assistant explicitly attributes the recurring image-transform 503s to the gateway/platform layer, noting they appear only in gateway logs with no matching function execution logs while nearby 200s have execution logs. It also distinguishes these gateway 503s from avatar-upload's in-function 500. Although it speculates about resource exhaustion and suggests reducing invocation load, the primary layer attribution is platform/gateway rather than application code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including pulling detailed edge-function metrics/boot logs around specific 503 timestamps, checking for worker/resource limit errors, reducing invocation resource use, adding retries, and increasing compute/limits if capacity-related." + "judgeNotes": "Recommended concrete next steps, including checking Edge Function metrics/dashboard for CPU/memory and WORKER_LIMIT around the 503 timestamps, plus specific mitigation actions." } ], "skills": { @@ -1264,7 +1288,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed deny-all RLS due to no policies and added authenticated SELECT and INSERT owner-scoped policies using auth.uid(), without disabling RLS." + "judgeNotes": "Diagnosed RLS enabled with no policies causing default-deny/zero Data API rows, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK. Also verified behavior." } ], "skills": { @@ -1328,7 +1352,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied through `supabase db push` (#14), which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local file `supabase/migrations/20240115000000_add_profile_bio.sql` (#12), after which `supabase migration list` showed local and remote aligned (#13) and the successful push proceeded. No disallowed workaround or direct mutation was used; psql commands were read-only inspection." + "judgeNotes": "Avatar migration was applied through `supabase db push` in action #15, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` History was reconciled by adding the missing local migration file `20240115000000_add_profile_bio.sql` in action #13, after which `supabase migration list` showed local/remote alignment and the subsequent CLI push succeeded. Read-only `psql` inspection was used, but no direct SQL mutation or prepared-statement workaround was seen." } ], "skills": { @@ -1385,7 +1409,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -1637,12 +1661,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 7) from the queue" + "notes": "function removed the seeded message (id 36) from the queue" } ], "skills": { @@ -1650,32 +1674,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job send message to queue pgmq\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 75189 - } - ] + "calls": [] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", @@ -1733,7 +1732,7 @@ }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-opus-4.8-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { @@ -1823,7 +1822,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "seed rows present", @@ -1833,42 +1832,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"unauthorized\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"14f44167-7395-4098-9ee4-65835041dd08\",\"metric\":\"steps_a_mrnzcw4x\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"643b8cc3-67bb-4091-9d27-f2e7d78066ed\",\"metric\":\"steps_a_mry6q4v5\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"14f44167-7395-4098-9ee4-65835041dd08\",\"metric\":\"steps_a_mrnzcw4x\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"643b8cc3-67bb-4091-9d27-f2e7d78066ed\",\"metric\":\"steps_a_mry6q4v5\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"1b50150f-6ab2-42bd-8008-f0c40d3404f9\",\"metric\":\"steps_b_mrnzcw4x\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"6f2bed62-fcbf-4b05-8215-2ddb9568eb7b\",\"metric\":\"steps_b_mry6q4v5\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"unauthorized\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"unauthorized\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"unauthorized\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "implementation uses @supabase/server", - "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -1879,67 +1878,59 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function dual authentication service role key user token verify jwt\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function verify apikey secret key service role authenticate user\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", "title": "Envoy API Gateway" } ], - "resultChars": 65709 + "resultChars": 106133 }, { "source": "search_docs", - "query": "{ securing: searchDocs(query: \"Securing Edge Functions service role bypass RLS API key apikey header pattern\", limit: 3) { nodes { title href content } } server: searchDocs(query: \"@supabase/server createClient edge function new API keys secret publishable\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Securing Edge Functions withSupabase auth mode array multiple user secret ctx supabaseAdmin authInfo\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" } ], - "resultChars": 46583 + "resultChars": 32364 } ] }, @@ -1986,7 +1977,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019f6c9d-4b62-7791-8378-3d0bd8ab5048/receipt-alpha.pdf, 019f6c9d-4b62-7791-8378-3d0bd8ab5048/receipt-beta.pdf" + "notes": "saw: 019f9166-12cf-747f-8d6a-ea197d01ca8d/receipt-alpha.pdf, 019f9166-12cf-747f-8d6a-ea197d01ca8d/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -2007,7 +1998,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no RLS disabling or public access, and uses createSignedUrl with expiry for sharing." + "judgeNotes": "The answer creates a private user-files bucket, adds authenticated-only SELECT and INSERT policies on storage.objects scoped to bucket_id and auth.uid() via first path segment, does not disable RLS or use permissive/public policies, and provides supabase-js createSignedUrl code with an expiry. It also correctly avoids service-role client use." } ], "skills": { @@ -2018,13 +2009,17 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy private bucket user folder owner access control\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policies private bucket user folder owner\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", "title": "Storage Buckets" @@ -2036,17 +2031,9 @@ { "url": "https://supabase.com/docs/guides/security/product-security", "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" } ], - "resultChars": 22916 + "resultChars": 23289 } ] }, @@ -2085,12 +2072,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "6 passed, 2 failed" + "notes": "4 passed, 2 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the broken tenant isolation policy, explains that authenticated members can read posts from organizations they are not members of, and grounds the conclusion in the pgTAP failures. It also distinguishes `notes` as correctly isolated." + "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members can read posts across organizations due to the missing `org_id` predicate, and grounds the conclusion in pgTAP failures while noting `notes` is correctly isolated." } ], "skills": { @@ -2199,12 +2186,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets requirements: preserves app scrape, adds HTTPS Supabase metrics scrape with correct path, Basic Auth using password_file, project target on supabase.co, and docker-compose mounts the secrets directory containing the password file." + "judgeNotes": "Prometheus preserves the app job and adds a Supabase scrape using HTTPS, the required metrics path, Basic Auth with password_file, and a .supabase.co target. docker-compose mounts ./secrets to the matching password_file path." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes concrete steps to replace project ref, create a Supabase Secret API key, place it in the mounted secrets/supabase_metrics_key file, reload/start the Compose stack, and verify via curl plus Prometheus targets." + "judgeNotes": "README includes required Secret API key creation, matching secret file placement, Compose/Prometheus reload/start steps, and concrete verification via curl plus Prometheus targets." } ], "skills": { @@ -2215,7 +2202,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project monitoring\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint observability project metrics\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -2233,15 +2220,23 @@ { "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", + "title": "How to View Database Metrics" } ], - "resultChars": 23542 + "resultChars": 33507 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-4.8-no-skills/deploy-database-001-prometheus-metrics.json" }, { @@ -2277,7 +2272,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -2368,7 +2363,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "victim session active before delete-account", @@ -2389,7 +2384,8 @@ }, { "name": "deleted user cannot sign back in", - "passed": true + "passed": false, + "notes": "deleted account can still sign in" }, { "name": "other users keep their sessions and access", @@ -2398,7 +2394,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Meets all rubric requirements: identifies soft-delete-only bug, implements real auth user deletion/session revocation, correctly explains stateless JWT residual window and aligns it with added RLS live-profile mitigation while caveating local validation, and accurately distinguishes publishable vs secret keys including RLS behavior and frontend/server placement." + "judgeNotes": "Diagnoses the soft-delete-only bug, implements real auth revocation via banning plus deleting sessions/refresh tokens and RLS enforcement, explains the remaining stateless JWT/local-validation window consistently, and correctly distinguishes publishable vs secret keys and RLS behavior." } ], "skills": { @@ -2406,40 +2402,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key API keys migration anon service_role RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - } - ], - "resultChars": 169611 - } - ] + "calls": [] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-4.8-no-skills/investigate-auth-001-deleted-user-access.json" }, { @@ -2488,7 +2455,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identifies that the channel reaches SUBSCRIBED but INSERT events do not arrive because public.orders is missing from the supabase_realtime publication. It applies exactly the required fix via ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verifies the publication, and explicitly leaves RLS/policies and courier_locations intact without blaming or weakening them." + "judgeNotes": "The assistant correctly identified the root cause as `orders` missing from the `supabase_realtime` publication despite the channel reaching SUBSCRIBED, added only `public.orders` to the existing publication, and did not weaken RLS/policies or disrupt `courier_locations`." } ], "skills": { @@ -2527,17 +2494,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as affected and described the recurring HTTP 503 gateway failures throughout the morning of 2026-04-28, covering the 8 failures from ~07:00Z to 12:00Z, while ruling out billing-webhook noise." + "judgeNotes": "Identified `image-transform` as the affected function and explicitly described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering all eight gateway failures from 07:00Z through 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes recurring image-transform 503s to the gateway/platform layer before the function, grounded in valid observations: 503s appear in gateway logs with no corresponding function execution 503s, executions that reached the function were 200s, and distinguishes the avatar-upload 500 as a separate function-level error." + "judgeNotes": "Attributes recurring 503s to the platform/gateway layer before the function code, grounded in valid observations: gateway/API logs show 503s with no matching execution logs, actual invocations all returned 200, deployment/version unchanged, and distinguishes this from the avatar-upload function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps including checking Edge Function resource limits/concurrency, correlating 503 timestamps with traffic spikes, and opening a Supabase support ticket referencing gateway 503s." + "judgeNotes": "The assistant recommended multiple concrete next steps, including checking boot-level Edge Function logs for specific 503 event IDs, mitigating cold starts by bundling dependencies or adding a warm-up cron, adding retries, and adjusting resource/timeout limits if indicated." } ], "skills": { @@ -2603,7 +2570,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "Diagnosed RLS enabled with no policies/default-deny, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { @@ -2662,7 +2629,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Applied the pending avatar_url migration with `supabase db push` in step #18; output shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding the local file `supabase/migrations/20240115000000_add_bio.sql` in step #16, after which migration list matched local/remote in step #17 and db push succeeded. No disallowed workaround observed." + "judgeNotes": "Avatar migration was applied through `supabase db push` in action #15, with output showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local migration file `20240115000000_add_profile_bio.sql` in action #13, after which `supabase migration list` showed local and remote aligned. No prohibited workaround or direct SQL mutation was used." } ], "skills": { @@ -2839,7 +2806,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -2857,7 +2824,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -2966,12 +2933,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -2987,31 +2954,107 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule pgmq send queue example\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job cron.schedule syntax\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, { "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", "title": "pg_cron debugging guide" }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + } + ], + "resultChars": 32862 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function reading from queue pgmq worker\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + } + ], + "resultChars": 65902 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase Queues pgmq read pop delete example edge function worker\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + } + ], + "resultChars": 8521 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"cron job invoke edge function vault secret service_role key best practice\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/functions/schedule-functions", "title": "Scheduling Edge Functions" + } + ], + "resultChars": 2208 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"config.toml functions verify_jwt per function local development\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" } ], - "resultChars": 46500 + "resultChars": 21389 } ] }, @@ -3186,17 +3229,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f3de9df-d36e-4fe3-afa6-585831ee3ab6\",\"metric\":\"steps_a_mrnza8oa\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"fbaa16d8-af77-4f9f-9135-141d3f43fe77\",\"metric\":\"steps_a_mry6m8z5\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f3de9df-d36e-4fe3-afa6-585831ee3ab6\",\"metric\":\"steps_a_mrnza8oa\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"fbaa16d8-af77-4f9f-9135-141d3f43fe77\",\"metric\":\"steps_a_mry6m8z5\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"547d02a8-3075-460e-b74b-3566ca328be3\",\"metric\":\"steps_b_mrnza8oa\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"66bbd10d-7d3f-4d7b-bee3-b0c13b464fb3\",\"metric\":\"steps_b_mry6m8z5\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -3206,7 +3249,7 @@ { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 504: { \"message\":\"The upstream server is timing out\" }" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", @@ -3232,35 +3275,55 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY secret key default secrets\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"functions new --auth apikey user none scaffold edge function\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + }, + { + "url": "https://supabase.com/docs/guides/ai/examples/openai", + "title": "Generating OpenAI GPT3 completions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/mcp", + "title": "Supabase MCP Server" + }, + { + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-functions-new", + "title": "Create a new Function locally" }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" + "url": "https://supabase.com/docs/reference/cli/supabase-functions", + "title": "Manage Supabase Edge functions" } ], - "resultChars": 36932 + "resultChars": 116247 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"verify_jwt config.toml edge functions per function\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function service_role key apikey header verify_jwt config.toml\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3268,51 +3331,249 @@ "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - } - ], - "resultChars": 27882 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Securing Edge Functions auth modes user secret combine multiple auth same function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/functions/auth-headers", "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 126870 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"new API keys publishable key secret key SUPABASE_SERVICE_ROLE_KEY environment variables edge functions\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" + }, + { + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" + } + ], + "resultChars": 187026 + }, + { + "source": "web_search", + "query": "supabase functions new --auth apikey user none flag CLI 2026", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/api/api-keys", + "title": "Understanding API keys | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/reference/cli/introduction", + "title": "CLI Reference | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/reference/server/types-supabaseenv", + "title": "Server API Reference | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-functions-new", + "title": "Supabase CLI reference - Create a function" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth | Supabase Docs" + } + ], + "resultChars": 2422 + }, + { + "source": "web_search", + "query": "supabase CLI functions new --auth flag edge function template", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/reference/cli/introduction", + "title": "CLI Reference | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/config", + "title": "Supabase CLI config | Supabase Docs" + }, + { + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture | Supabase Docs" + }, + { + "url": "https://supabase.com/edge-functions", + "title": "Edge Functions | Supabase" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard) | Supabase Docs" + } + ], + "resultChars": 2305 + }, + { + "source": "web_fetch", + "query": "Extract everything about: the --auth flag for supabase functions new (none, apikey, user modes) and what code each scaffolds; how to check apikey header against service_role key; how to distinguish anon+JWT caller vs service_role caller; env var names used (SUPABASE_URL, SUPABASE_ANON_KEY, SUPABASE_SERVICE_ROLE_KEY, or new publishable/secret key names); verify_jwt config.toml setting and its effect on apikey-only callers. Include exact code snippets and header names.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth.md" } ], - "resultChars": 33453 + "resultChars": 1600 + }, + { + "source": "web_fetch", + "query": "Extract the full CLI reference for 'supabase functions new', including the --auth flag, its allowed values (none, apikey, user), and description of what each generates.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/cli/supabase-functions-new.md" + } + ], + "resultChars": 1007 + }, + { + "source": "web_fetch", + "query": "Extract: new publishable and secret API key naming; new env var names for Edge Functions (replacing SUPABASE_ANON_KEY / SUPABASE_SERVICE_ROLE_KEY); whether old env vars still work; recommended way to check if a caller is using the secret/service-role key in an edge function; any mention of '@supabase/server API-key env' or similar migration helper.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" + } + ], + "resultChars": 1338 + }, + { + "source": "web_fetch", + "query": "Extract everything about the verify_jwt setting in config.toml for edge functions: what it does, whether it affects apikey header validation vs Authorization Bearer JWT validation, and how service_role callers interact with it.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/function-configuration.md" + } + ], + "resultChars": 1134 + }, + { + "source": "web_fetch", + "query": "Extract everything about Authorization headers, apikey header, how Supabase client SDK sends apikey and Authorization headers, and how Edge Functions gateway processes verify_jwt against these headers. Include exact header names and behavior for service_role vs anon vs user JWT.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-headers.md" + } + ], + "resultChars": 1559 + }, + { + "source": "web_fetch", + "query": "Give me the exact flags table and description for 'supabase functions new', especially the --auth flag: what values are accepted (none, apikey, user) and what each value does / what code template it generates. Quote exact text if possible.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/cli/supabase-functions-new" + } + ], + "resultChars": 826 } ] }, @@ -3359,7 +3620,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019f6c9d-2621-71dc-bf54-b0321882d158/receipt-alpha.pdf, 019f6c9d-2621-71dc-bf54-b0321882d158/receipt-beta.pdf" + "notes": "saw: 019f9165-dceb-73e2-a03b-91884c7a86a1/receipt-alpha.pdf, 019f9165-dceb-73e2-a03b-91884c7a86a1/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -3380,7 +3641,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates a private user-files bucket, owner-scoped SELECT and INSERT policies on storage.objects for authenticated users using the user-id path prefix, does not disable RLS or make the bucket public, and provides supabase-js createSignedUrl code with an expiry for temporary sharing." + "judgeNotes": "Meets the rubric: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects using folder prefix auth.uid(), no RLS disabling or public access, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -3396,55 +3657,68 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy folder path user id owner\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"storage RLS policy restrict access to a folder by user id\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { "url": "https://supabase.com/docs/guides/storage/security/ownership", "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", + "title": "Error Codes" } ], - "resultChars": 64123 + "resultChars": 35750 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiring link\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"create bucket SQL migration storage.buckets insert private bucket\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/self-hosting/copy-from-platform-s3", + "title": "Copy Storage Objects from Platform" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" }, { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" + } + ], + "resultChars": 69625 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"create signed url temporary sharing createSignedUrl\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" } ], - "resultChars": 7426 + "resultChars": 2612 } ] }, @@ -3478,17 +3752,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "4 file(s): supabase/tests/database/memberships_exposure.test.sql, supabase/tests/database/posts_tenant_isolation.test.sql, supabase/tests/database/00_rls_enabled.test.sql, supabase/tests/database/notes_tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "2 passed, 1 failed" + "notes": "7 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, explains that authenticated members can read posts from other organizations due to the missing `org_id` match, and grounds the conclusion in the pgTAP failures while noting `notes` passed." + "judgeNotes": "The agent correctly identifies posts as having the tenant isolation flaw, grounded in pgTAP failures showing cross-org post reads. It treats test results as authoritative and distinguishes notes as correctly isolated for reads." } ], "skills": { @@ -3501,36 +3775,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pgTAP testing RLS policies auth.uid() local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 75875 - } - ] + "calls": [] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -3602,56 +3847,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"match_document_sections semantic search gte-small embedding\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" - }, - { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" - }, - { - "url": "https://supabase.com/docs/guides/ai/concepts", - "title": "Concepts" - } - ], - "resultChars": 65363 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"gte-small model dimensions Supabase.ai Session embedding edge function\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, - { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" - } - ], - "resultChars": 37366 - } - ] + "calls": [] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -3685,12 +3881,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, preserves the app job, and docker-compose mounts the secrets directory containing the password_file." + "judgeNotes": "Adds HTTPS Supabase scrape at /customer/v1/privileged/metrics for .supabase.co:443 using basic_auth with password_file, preserves app scrape, and docker-compose mounts the secrets directory containing the password file." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, Prometheus reload, and concrete verification via Prometheus targets, PromQL, and Grafana." + "judgeNotes": "README.md includes live setup steps: create a Supabase Secret API key, write it to the expected secret file, replace project ref placeholders, restart/reload the Compose stack, and verify via direct endpoint curl, Prometheus targets, and Grafana dashboard." } ], "skills": { @@ -3731,34 +3927,6 @@ } ], "resultChars": 27147 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"metrics customer/v1/privileged/metrics\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - } - ], - "resultChars": 54724 } ] }, @@ -3800,7 +3968,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -3875,16 +4043,44 @@ }, "docs": { "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosting with Docker docker-compose setup\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + } + ], + "resultChars": 91121 + }, { "source": "web_fetch", - "query": "List the exact steps to set up self-hosted Supabase with Docker: which repo/files to clone or copy, the docker-compose.yml structure, which env vars need to be set in .env (list all of them with descriptions), and how JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, POSTGRES_PASSWORD, DASHBOARD credentials, SECRET_KEY_BASE, VAULT_ENC_KEY are generated/used.", + "query": "Extract the full step-by-step instructions for self-hosting Supabase with Docker, including the git clone/sparse checkout commands, the .env setup process, docker-compose commands, and any notes about generating secrets (JWT secret, API keys, postgres password, dashboard credentials).", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker" + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 3336 + "resultChars": 3491 } ] }, @@ -3913,7 +4109,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "victim session active before delete-account", @@ -3934,7 +4130,8 @@ }, { "name": "deleted user cannot sign back in", - "passed": true + "passed": false, + "notes": "deleted account can still sign in" }, { "name": "other users keep their sessions and access", @@ -3943,7 +4140,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Meets all rubric requirements: correctly diagnoses soft-delete-only flow, implements hard auth user deletion/session revocation, accurately explains residual JWT access window consistent with the implemented fix and names mitigations, and correctly distinguishes publishable vs secret keys and RLS behavior." + "judgeNotes": "Diagnoses the soft-delete-only flow, implements real revocation via auth.sessions deletion and auth.users ban, adds RLS checks to close the Data API stale-JWT window, and consistently explains that JWTs remain cryptographically valid until expiry for local validation. Correctly distinguishes publishable/anon frontend use from secret/service_role server-only BYPASSRLS behavior." } ], "skills": { @@ -3956,7 +4153,73 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"delete user account self deletion RPC function auth.users best practice\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + }, + { + "url": "https://supabase.com/docs/reference/python/auth-admin-deleteuser", + "title": "delete_user()" + } + ], + "resultChars": 8456 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable and secret API keys vs anon and service_role RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/refine", + "title": "Use Supabase with Refine" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + } + ], + "resultChars": 83270 + }, + { + "source": "web_fetch", + "query": "Summarize: what are publishable and secret API keys, how do they map to legacy anon and service_role keys, which one goes in frontend code, and what does this mean for Row Level Security enforcement?", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/api-keys.md" + } + ], + "resultChars": 1225 + } + ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", @@ -4004,12 +4267,12 @@ { "name": "staff can still read orders through RLS", "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "notes": "authenticated sees 3 of 3 orders" }, { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Diagnosed the missing orders table in supabase_realtime publication, applied ALTER PUBLICATION ADD TABLE public.orders, preserved courier_locations/RLS/policies, and did not blame or alter unrelated components." + "judgeNotes": "The answer correctly identifies the root cause as orders missing from the supabase_realtime publication despite SUBSCRIBED status, applies ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserves RLS/policies and courier_locations. It does not blame RLS/client/networking or weaken security." } ], "skills": { @@ -4053,17 +4316,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant named image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering the eight gateway failures from 07:00Z through 12:00Z. It also distinguished this from the older billing-webhook issue." + "judgeNotes": "The assistant clearly identified `image-transform` as the affected function and described a recurring pattern of eight HTTP 503 gateway failures across the morning of 2026-04-28, listing times from 07:00Z through 12:00Z. It did not incorrectly focus on billing-webhook or remain vague." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the recurring image-transform 503s to the API gateway/platform layer before function code ran, grounded in the observation that 503 entries appear only in gateway logs with no execution_time_ms/deployment_id/version while nearby 200s succeeded. Also distinguishes these gateway 503s from avatar-upload's function-level 500." + "judgeNotes": "Attributes recurring image-transform 503s to gateway/worker/platform layer before the handler, not function application code. Grounds this in valid observations: 503s absent from edge-function invocation/runtime logs while nearby 200s appear, and distinguishes gateway 503s from an avatar-upload function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete next steps including adding retry/backoff, reducing cold-start frequency, improving alerting by gateway vs runtime failures, and investigating slow initialization in the private package." + "judgeNotes": "The assistant recommended concrete next steps, including confirming resource/runtime metrics with Supabase support/dashboard, reducing cold-start weight, adding retries and alerts, and checking deployment history." } ], "skills": { @@ -4134,7 +4397,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results; kept RLS enabled; created authenticated SELECT policy owner-scoped by user_id = auth.uid(); created authenticated INSERT policy with WITH CHECK enforcing user_id = auth.uid()." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid() with WITH CHECK for inserts." } ], "skills": { @@ -4198,7 +4461,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push` in #25, which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled orphan bio migration by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #23, after which `supabase migration list` showed local and remote aligned in #24. Direct `psql` commands were read-only inspection; no prohibited workaround seen." + "judgeNotes": "Avatar migration was applied by `supabase db push` in #10, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #8, after which `supabase migration list` showed local/remote aligned and the subsequent CLI push succeeded. No disallowed direct SQL mutation or prepared-statement workaround was used; psql was read-only inspection only." } ], "skills": { @@ -4390,7 +4653,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -4408,7 +4671,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -4512,7 +4775,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -4520,7 +4783,64 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron schedule pgmq send queue example\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + } + ], + "resultChars": 46500 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"local development invoke edge function from pg_cron docker network host.docker.internal kong\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/webhooks", + "title": "Database Webhooks" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 42177 + } + ] }, "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", @@ -4683,17 +5003,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"470e6d3d-5da0-4456-ac30-a58f189b2226\",\"metric\":\"steps_a_mrnyzk12\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"8f879357-1b1b-42c5-8e57-1b698204232a\",\"metric\":\"steps_a_mry6g9jy\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"470e6d3d-5da0-4456-ac30-a58f189b2226\",\"metric\":\"steps_a_mrnyzk12\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"8f879357-1b1b-42c5-8e57-1b698204232a\",\"metric\":\"steps_a_mry6g9jy\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"d5cf03db-1898-4d2d-8721-801cafe8f82a\",\"metric\":\"steps_b_mrnyzk12\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"c2218cee-ec97-456a-8f4f-8733a8b93c9e\",\"metric\":\"steps_b_mry6g9jy\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -4724,93 +5044,92 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server edge function new API keys secret key publishable key\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function service role key apikey header user_id trusted backend Deno.serve auth.getUser\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/http-methods", + "title": "Routing" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" } ], - "resultChars": 55720 + "resultChars": 39440 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server edge function sb-api-key header createClient\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"withSupabase auth mode array multiple auth user secret ctx.supabaseAdmin ctx.userClaims optional auth\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-getclient" + "url": "https://supabase.com/docs/guides/troubleshooting/migrating-auth-users-between-projects", + "title": "Migrating Auth Users Between Supabase Projects" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-deleteclient" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" } ], - "resultChars": 13437 + "resultChars": 37970 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions authentication verify_jwt service_role apikey header user JWT\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Securing Edge Functions every auth mode use case\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/database/overview", + "title": "Database" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" } ], - "resultChars": 75098 + "resultChars": 36472 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json" }, { @@ -4851,7 +5170,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019f6c9c-ff09-73dd-9a80-8ae97a305e9f/receipt-alpha.pdf, 019f6c9c-ff09-73dd-9a80-8ae97a305e9f/receipt-beta.pdf" + "notes": "saw: 019f9165-8fd0-70da-af6a-34b962344d00/receipt-alpha.pdf, 019f9165-8fd0-70da-af6a-34b962344d00/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -4872,7 +5191,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no RLS disabling or public access, and signed URL sharing via createSignedUrl with expiry." + "judgeNotes": "Configures a private user-files bucket, owner-scoped authenticated SELECT and INSERT policies on storage.objects with WITH CHECK for uploads, does not disable RLS, and uses createSignedUrl with an expiry for temporary sharing." } ], "skills": { @@ -4912,17 +5231,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/rls_tenant_isolation_test.sql" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "9 passed, 3 failed" + "notes": "4 passed, 2 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies posts as having the broken tenant isolation policy, specifically that members of any org can read posts from other orgs because the policy checks only membership existence and not org_id. It grounds this in the pgTAP failure (test 8) and treats the test results as authoritative. It also discusses memberships, but does not blame notes and explicitly says notes isolation passes." + "judgeNotes": "The agent correctly identifies `posts` as the table with the broken tenant isolation policy, explains that authenticated members can read posts from other organizations because the policy lacks `m.org_id = posts.org_id`, and grounds the conclusion in pgTAP results showing `posts` failures while `notes` passes." } ], "skills": { @@ -5031,12 +5350,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets requirements: preserves app scrape, adds HTTPS Supabase metrics endpoint with correct path and project target, uses HTTP Basic Auth with password_file, and docker-compose mounts the secrets directory containing that password file." + "judgeNotes": "Prometheus preserves the app scrape and adds a Supabase scrape using HTTPS, the required metrics path, Basic Auth with password_file, and a .supabase.co target. docker-compose mounts the secrets directory containing the referenced password_file." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes required live setup steps: create Secret API key, place it in the mounted secret file, replace project ref, and restart/reload Compose/Prometheus. It also provides concrete verification via Prometheus Targets, direct curl, and Grafana dashboard." + "judgeNotes": "README includes steps to create a Supabase Secret API key, place it in the Prometheus-mounted secret file, replace project ref, restart/recreate the Compose Prometheus service, and verify via Prometheus targets, curl, and Grafana dashboard." } ], "skills": { @@ -5204,7 +5523,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "victim session active before delete-account", @@ -5225,8 +5544,7 @@ }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", @@ -5235,7 +5553,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer diagnoses the soft-delete-only bug, implements real session/refresh-token revocation and blocks future sign-in, explains JWT expiry and the remaining/local-validation window consistently with its RLS mitigation, and correctly states publishable keys are client-safe while secret/service-role keys are server-only and bypass RLS. There is one slightly contradictory closing phrase about RLS being the enforcement layer regardless of key, but the surrounding explanation clearly says secret keys bypass RLS." + "judgeNotes": "The answer identifies the original soft-delete-only bug, implements auth.users deletion with session/refresh-token revocation via cascades, consistently explains the remaining stateless JWT access-token window and mitigation, and correctly distinguishes publishable/frontend/RLS behavior from secret/server-only/BYPASSRLS behavior." } ], "skills": { @@ -5247,7 +5565,7 @@ }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/investigate-auth-001-deleted-user-access.json" }, { @@ -5296,7 +5614,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Diagnosed the missing orders table in supabase_realtime publication, added public.orders to the existing publication, preserved courier_locations and RLS/policies, and did not blame or alter unrelated areas." + "judgeNotes": "The assistant correctly identified that the subscription reached SUBSCRIBED but no INSERT events arrived because public.orders was missing from the supabase_realtime publication. It fixed exactly that with `alter publication supabase_realtime add table public.orders;`, preserved courier_locations, and did not weaken RLS or policies." } ], "skills": { @@ -5335,17 +5653,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant identified `image-transform` as the main affected function and described a recurring morning pattern of gateway HTTP 503s on 2026-04-28 with timestamps spread from 07:00Z to 12:00Z. Although it listed five rather than all eight failures, it recognized the correct function and recurring pattern, satisfying the pass criteria." + "judgeNotes": "Identified `image-transform` as affected and described recurring HTTP 503 gateway failures throughout the morning of 2026-04-28, while distinguishing old billing-webhook 503s as separate." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes recurring image-transform 503s to gateway/infrastructure layer, grounded in observation that 503s appear only in API gateway logs with no corresponding edge-function execution logs, while nearby requests succeeded. It also distinguishes avatar-upload's function-level 500 from the gateway 503s." + "judgeNotes": "Attributes recurring image-transform 503s to the gateway/platform layer, not function code, and grounds this in valid observations: 503s absent from invocation logs while 200s appear, requests rejected before runtime, unchanged deployment version, and distinction from avatar-upload's in-function 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: checking bundle/init cost, adding warm-up pings and retries, investigating the separate function error, and adding monitoring/alerting. These are specific actionable steps beyond vague log checking." + "judgeNotes": "The assistant provided concrete actionable next steps, including checking concurrency/rate limits, adding structured logging, implementing retries, and investigating the specific 09:00 in-function 500." } ], "skills": { @@ -5411,7 +5729,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all for the Data API, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), using WITH CHECK for INSERT." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to auth.uid() = user_id with WITH CHECK for inserts." } ], "skills": { @@ -5470,7 +5788,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied by `supabase db push` in #19, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the matching local migration file `20240115000000_add_profile_bio.sql` in #17, after which `supabase migration list` in #18 showed local and remote aligned. Only read-only psql inspection was used; no direct SQL mutation or prepared-statement workaround was seen." + "judgeNotes": "Avatar migration was applied through the Supabase CLI with `supabase db push` in #15, showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `20240115000000_add_profile_bio.sql` in #13, after which CLI migration list showed local/remote aligned and the successful push proceeded. Read-only psql inspection was used; no prohibited direct SQL mutation or prepared-statement workaround was seen." } ], "skills": { @@ -5680,906 +5998,732 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"local development migrations RLS insert policies seed data Data API exposed table\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api#add-rls-policies", - "title": "Add RLS policies" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api#grant-access-explicitly", - "title": "Grant access explicitly" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api#use-a-dedicated-api-schema", - "title": "Use a dedicated API schema" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api#default-privileges-for-new-tables-and-functions", - "title": "Default privileges for new tables and functions" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api#accessing-request-information", - "title": "Accessing request information" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api#enforce-additional-rules-on-each-request", - "title": "Enforce additional rules on each request" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api#disable-the-data-api", - "title": "Disable the Data API" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-start-the-local-stack", - "title": "Step 2: Start the local stack" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-create-your-schema", - "title": "Step 3: Create your schema" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-add-seed-data", - "title": "Step 4: Add seed data" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-verify", - "title": "Step 5: Verify" - }, + "source": "web_search", + "query": "site:supabase.com changelog.md Supabase breaking-change RLS Data API 2026", + "pages": [] + } + ] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"local development migrations add column table\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-commit", - "title": "Step 6: Commit" + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-daily-workflow", - "title": "The daily workflow" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#making-schema-changes", - "title": "Making schema changes" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#generating-types", - "title": "Generating types" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#staying-in-sync-with-your-team", - "title": "Staying in sync with your team" - }, + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + } + ], + "resultChars": 70469 + } + ] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 36) from the queue" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pg_cron schedule cron job Supabase enqueue task queue pgmq pop send edge function\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ClientLibraryFunctionReference { title href methodName language content href } ... on TroubleshootingGuide { title href content } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#pushing-to-a-remote-project", - "title": "Pushing to a remote project" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#resetting-a-remote-dev-or-staging-project", - "title": "Resetting a remote dev or staging project" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#key-commands-at-a-glance", - "title": "Key commands at a glance" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#cleaning-up-generated-migrations", - "title": "Cleaning up generated migrations" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#grants", - "title": "Grants" - }, + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + } + ], + "resultChars": 27960 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pgmq queue pop send delete Supabase queues expose via postgrest\", limit: 10) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href methodName language content href } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#revokere-grant-patterns", - "title": "Revoke/re-grant patterns" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#extension-statements", - "title": "Extension statements" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#known-limitations-of-db-diff", - "title": "Known limitations of db diff" + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-supabase-directory", - "title": "The ./supabase directory" + "url": "https://supabase.com/docs/guides/api", + "title": "Data REST API" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#move-an-existing-project-to-local-development", - "title": "Move an existing project to local development" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize", - "title": "Step 1: Initialize" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-authenticate", - "title": "Step 2: Authenticate" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-link-to-your-remote-project", - "title": "Step 3: Link to your remote project" - }, + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" + } + ], + "resultChars": 90610 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pgmq list queues create if not exists queue exists function\", limit: 10) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href methodName language content href } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-pull-the-remote-schema", - "title": "Step 4: Pull the remote schema" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-create-seed-data", - "title": "Step 5: Create seed data" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-verify", - "title": "Step 6: Verify" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-7-commit", - "title": "Step 7: Commit" - }, + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + } + ], + "resultChars": 66929 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Edge Functions environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY local runtime\", limit: 10) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href methodName language content href } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#start-a-new-project-from-scratch", - "title": "Start a new project from scratch" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize-1", - "title": "Step 1: Initialize" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#more-information", - "title": "More information" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#frontend-access", - "title": "Frontend access" + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#direct-database-connections", - "title": "Direct database connections" + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#edge-functions", + "url": "https://supabase.com/docs/guides/functions", "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#data-api", - "title": "Data API" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data#connecting-your-app-securely", - "title": "Connecting your app securely" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres", - "title": "Connect to your database" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#direct-connection", - "title": "Direct connection" + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#data-apis-and-client-libraries", - "title": "Data APIs and client libraries" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#quickstarts", - "title": "Quickstarts" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#how-to-connect-to-your-postgres-databases", - "title": "How to connect to your Postgres databases" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#what-is-the-max-pooler-clients-limit", - "title": "What is the max pooler clients limit?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#how-to-choose-the-right-connection-method", - "title": "How to choose the right connection method?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#does-connection-pooling-affect-latency", - "title": "Does connection pooling affect latency?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#why-do-connection-strings-have-different-ports", - "title": "Why do connection strings have different ports?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#why-are-there-active-connections-when-the-app-is-idle", - "title": "Why are there active connections when the app is idle?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#where-can-you-see-current-connection-usage", - "title": "Where can you see current connection usage?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#what-is-the-difference-between-client-connections-and-backend-connections", - "title": "What is the difference between client connections and backend connections?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#how-does-the-default-pool-size-work", - "title": "How does the default pool size work?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#can-you-use-supavisor-and-pgbouncer-together", - "title": "Can you use Supavisor and PgBouncer together?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#where-is-the-postgres-connection-string-in-supabase", - "title": "Where is the Postgres connection string in Supabase?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#how-do-you-connect-using-ipv4", - "title": "How do you connect using IPv4?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#what-is-the-fatal-password-authentication-failed-error", - "title": "What is the “FATAL: Password authentication failed” error?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#what-is-a-connection-refused-error", - "title": "What is a “connection refused” error?" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#troubleshooting-and-postgres-connection-string-faqs", - "title": "Troubleshooting and Postgres connection string FAQs" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#connecting-with-ssl", - "title": "Connecting with SSL" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#server-side-poolers", - "title": "Server-side poolers" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#application-side-poolers", - "title": "Application-side poolers" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#more-about-connection-pooling", - "title": "More about connection pooling" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#dedicated-pooler", - "title": "Dedicated pooler" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#pooler-transaction-mode", - "title": "Pooler transaction mode" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#pooler-session-mode", - "title": "Pooler session mode" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres#poolers", - "title": "Poolers" - }, - { - "url": "https://supabase.com/docs/guides/storage/production/scaling", - "title": "Storage Optimizations" - }, - { - "url": "https://supabase.com/docs/guides/storage/production/scaling#egress", - "title": "Egress" - }, - { - "url": "https://supabase.com/docs/guides/storage/production/scaling#optimizing-rls", - "title": "Optimizing RLS" - }, - { - "url": "https://supabase.com/docs/guides/storage/production/scaling#optimize-listing-objects", - "title": "Optimize listing objects" - }, - { - "url": "https://supabase.com/docs/guides/storage/production/scaling#smart-cdn", - "title": "Smart CDN" - }, - { - "url": "https://supabase.com/docs/guides/storage/production/scaling#limit-the-upload-size", - "title": "Limit the upload size" - }, - { - "url": "https://supabase.com/docs/guides/storage/production/scaling#set-a-high-cache-control-value", - "title": "Set a high cache-control value" - }, - { - "url": "https://supabase.com/docs/guides/storage/production/scaling#resize-images", - "title": "Resize images" + "url": "https://supabase.com/docs/guides/integrations/vercel-marketplace", + "title": "Vercel Marketplace" } ], - "resultChars": 180379 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com changelog.md supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "\"https://supabase.com/changelog.md\"", - "pages": [] - }, + "resultChars": 123642 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=403" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=403" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI init migration new seed local development\", limit: 10) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Edge Functions auth getUser verify JWT private notes secure access control\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-supabase-directory", - "title": "The ./supabase directory" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#move-an-existing-project-to-local-development", - "title": "Move an existing project to local development" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize", - "title": "Step 1: Initialize" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-authenticate", - "title": "Step 2: Authenticate" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", + "title": "Fetching the user" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-link-to-your-remote-project", - "title": "Step 3: Link to your remote project" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", + "title": "Setting up auth context" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-pull-the-remote-schema", - "title": "Step 4: Pull the remote schema" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", + "title": "Example" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-create-seed-data", - "title": "Step 5: Create seed data" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-verify", - "title": "Step 6: Verify" + "url": "https://supabase.com/docs/guides/functions/auth-headers#understanding-authorization-headers", + "title": "Understanding authorization headers" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-7-commit", - "title": "Step 7: Commit" + "url": "https://supabase.com/docs/guides/functions/auth-headers#the-verify_jwt-platform-check", + "title": "The verify_jwt platform check" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#start-a-new-project-from-scratch", - "title": "Start a new project from scratch" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize-1", - "title": "Step 1: Initialize" + "url": "https://supabase.com/docs/guides/getting-started/features#phone-logins", + "title": "Phone logins" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-start-the-local-stack", - "title": "Step 2: Start the local stack" + "url": "https://supabase.com/docs/guides/getting-started/features#passwordless-login", + "title": "Passwordless login" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-create-your-schema", - "title": "Step 3: Create your schema" + "url": "https://supabase.com/docs/guides/getting-started/features#authorization-via-row-level-security", + "title": "Authorization via Row Level Security" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-add-seed-data", - "title": "Step 4: Add seed data" + "url": "https://supabase.com/docs/guides/getting-started/features#captcha-protection", + "title": "CAPTCHA protection" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-verify", - "title": "Step 5: Verify" + "url": "https://supabase.com/docs/guides/getting-started/features#server-side-auth", + "title": "Server-Side Auth" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-commit", - "title": "Step 6: Commit" + "url": "https://supabase.com/docs/guides/getting-started/features#storage", + "title": "Storage" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-daily-workflow", - "title": "The daily workflow" + "url": "https://supabase.com/docs/guides/getting-started/features#file-storage", + "title": "File storage" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#making-schema-changes", - "title": "Making schema changes" + "url": "https://supabase.com/docs/guides/getting-started/features#content-delivery-network", + "title": "Content Delivery Network" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#generating-types", - "title": "Generating types" + "url": "https://supabase.com/docs/guides/getting-started/features#smart-content-delivery-network", + "title": "Smart Content Delivery Network" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#staying-in-sync-with-your-team", - "title": "Staying in sync with your team" + "url": "https://supabase.com/docs/guides/getting-started/features#image-transformations", + "title": "Image transformations" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#pushing-to-a-remote-project", - "title": "Pushing to a remote project" + "url": "https://supabase.com/docs/guides/getting-started/features#resumable-uploads", + "title": "Resumable uploads" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#resetting-a-remote-dev-or-staging-project", - "title": "Resetting a remote dev or staging project" + "url": "https://supabase.com/docs/guides/getting-started/features#s3-compatibility", + "title": "S3 compatibility" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#key-commands-at-a-glance", - "title": "Key commands at a glance" + "url": "https://supabase.com/docs/guides/getting-started/features#edge-functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#cleaning-up-generated-migrations", - "title": "Cleaning up generated migrations" + "url": "https://supabase.com/docs/guides/getting-started/features#deno-edge-functions", + "title": "Deno Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#grants", - "title": "Grants" + "url": "https://supabase.com/docs/guides/getting-started/features#regional-invocations", + "title": "Regional invocations" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#revokere-grant-patterns", - "title": "Revoke/re-grant patterns" + "url": "https://supabase.com/docs/guides/getting-started/features#npm-compatibility", + "title": "NPM compatibility" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#extension-statements", - "title": "Extension statements" + "url": "https://supabase.com/docs/guides/getting-started/features#project-management", + "title": "Project management" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#known-limitations-of-db-diff", - "title": "Known limitations of db diff" + "url": "https://supabase.com/docs/guides/getting-started/features#cli", + "title": "CLI" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/getting-started/features#management-api", + "title": "Management API" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/getting-started/features#client-libraries", + "title": "Client libraries" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/getting-started/features#feature-status", + "title": "Feature status" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#add-sample-data", - "title": "Add sample data" + "url": "https://supabase.com/docs/guides/getting-started/features#private-alpha", + "title": "Private alpha" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-your-project", - "title": "Deploy your project" + "url": "https://supabase.com/docs/guides/getting-started/features#public-alpha", + "title": "Public alpha" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#diffing-changes", - "title": "Diffing changes" + "url": "https://supabase.com/docs/guides/getting-started/features#generally-available", + "title": "Generally available" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#log-in-to-the-supabase-cli", - "title": "Log in to the Supabase CLI" + "url": "https://supabase.com/docs/guides/getting-started/features#beta", + "title": "Beta" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#link-your-project", - "title": "Link your project" + "url": "https://supabase.com/docs/guides/getting-started/features#database", + "title": "Database" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-database-changes", - "title": "Deploy database changes" + "url": "https://supabase.com/docs/guides/getting-started/features#postgres-database", + "title": "Postgres database" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-edge-functions", - "title": "Deploy Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/features#vector-database", + "title": "Vector database" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#use-auth-locally", - "title": "Use Auth locally" + "url": "https://supabase.com/docs/guides/getting-started/features#auto-generated-rest-api-via-postgrest", + "title": "Auto-generated REST API via PostgREST" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#sync-storage-buckets", - "title": "Sync storage buckets" + "url": "https://supabase.com/docs/guides/getting-started/features#auto-generated-graphql-api-via-pg_graphql", + "title": "Auto-generated GraphQL API via pg_graphql" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#sync-any-schema-with---schema", - "title": "Sync any schema with --schema" + "url": "https://supabase.com/docs/guides/getting-started/features#database-webhooks", + "title": "Database webhooks" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#limitations-and-considerations", - "title": "Limitations and considerations" + "url": "https://supabase.com/docs/guides/getting-started/features#secrets-and-encryption", + "title": "Secrets and encryption" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations#database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/getting-started/features#replication", + "title": "Replication" }, { - "url": "https://supabase.com/docs/guides/local-development/seeding-your-database", - "title": "Seeding your database" + "url": "https://supabase.com/docs/guides/getting-started/features#platform", + "title": "Platform" }, { - "url": "https://supabase.com/docs/guides/local-development/seeding-your-database#generating-seed-data", - "title": "Generating seed data" + "url": "https://supabase.com/docs/guides/getting-started/features#database-backups", + "title": "Database backups" }, { - "url": "https://supabase.com/docs/guides/local-development/seeding-your-database#what-is-seed-data", - "title": "What is seed data?" + "url": "https://supabase.com/docs/guides/getting-started/features#custom-domains", + "title": "Custom domains" }, { - "url": "https://supabase.com/docs/guides/local-development/seeding-your-database#using-seed-files", - "title": "Using seed files" + "url": "https://supabase.com/docs/guides/getting-started/features#network-restrictions", + "title": "Network restrictions" }, { - "url": "https://supabase.com/docs/guides/local-development/seeding-your-database#splitting-up-your-seed-file", - "title": "Splitting up your seed file" + "url": "https://supabase.com/docs/guides/getting-started/features#ssl-enforcement", + "title": "SSL enforcement" }, { - "url": "https://supabase.com/docs/guides/cli", - "title": "Local Dev with CLI" + "url": "https://supabase.com/docs/guides/getting-started/features#branching", + "title": "Branching" }, { - "url": "https://supabase.com/docs/guides/cli#resources", - "title": "Resources" + "url": "https://supabase.com/docs/guides/getting-started/features#terraform-provider", + "title": "Terraform provider" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" + "url": "https://supabase.com/docs/guides/getting-started/features#read-replicas", + "title": "Read replicas" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" + "url": "https://supabase.com/docs/guides/getting-started/features#log-drains", + "title": "Log drains" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" + "url": "https://supabase.com/docs/guides/getting-started/features#studio", + "title": "Studio" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" + "url": "https://supabase.com/docs/guides/getting-started/features#studio-single-sign-on", + "title": "Studio Single Sign-On" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" + "url": "https://supabase.com/docs/guides/getting-started/features#realtime", + "title": "Realtime" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" + "url": "https://supabase.com/docs/guides/getting-started/features#postgres-changes", + "title": "Postgres changes" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" + "url": "https://supabase.com/docs/guides/getting-started/features#broadcast", + "title": "Broadcast" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" + "url": "https://supabase.com/docs/guides/getting-started/features#presence", + "title": "Presence" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" + "url": "https://supabase.com/docs/guides/getting-started/features#auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" + "url": "https://supabase.com/docs/guides/getting-started/features#email-login", + "title": "Email login" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" + "url": "https://supabase.com/docs/guides/getting-started/features#social-login", + "title": "Social login" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" + "url": "https://supabase.com/docs/guides/functions#how-it-works", + "title": "How it works" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" + "url": "https://supabase.com/docs/guides/functions#quick-technical-notes", + "title": "Quick technical notes" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" + "url": "https://supabase.com/docs/guides/functions#when-to-use-edge-functions", + "title": "When to use Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" + "url": "https://supabase.com/docs/guides/functions#examples", + "title": "Examples" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" + "url": "https://supabase.com/docs/guides/database/secure-data#connecting-your-app-securely", + "title": "Connecting your app securely" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" + "url": "https://supabase.com/docs/guides/database/secure-data#data-api", + "title": "Data API" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" + "url": "https://supabase.com/docs/guides/database/secure-data#edge-functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" + "url": "https://supabase.com/docs/guides/database/secure-data#direct-database-connections", + "title": "Direct database connections" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/database/secure-data#frontend-access", + "title": "Frontend access" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration", - "title": "GitHub integration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#preparing-your-git-repository", - "title": "Preparing your Git repository" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#set-the-working-directory", - "title": "Set the working directory" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#installation", - "title": "Installation" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#email-notifications", - "title": "Email notifications" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#preventing-migration-failures", - "title": "Preventing migration failures" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#deploying-changes-to-production", - "title": "Deploying changes to production" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#seeding", - "title": "Seeding" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#migrations", - "title": "Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#syncing-github-branches", - "title": "Syncing GitHub branches" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#installing-the-supabase-cli", - "title": "Installing the Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#beta-channel", - "title": "Beta channel" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#learn-more", - "title": "Learn more" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#how-to-opt-out", - "title": "How to opt out" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#telemetry", - "title": "Telemetry" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#stopping-local-services", - "title": "Stopping local services" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#access-your-projects-services", - "title": "Access your project's services" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#running-supabase-locally", - "title": "Running Supabase locally" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#updating-the-supabase-cli", - "title": "Updating the Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#set-up-a-local-environment", - "title": "Set up a local environment" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#release-to-production", - "title": "Release to production" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#open-a-pr-with-new-migration", - "title": "Open a PR with new migration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#configure-github-actions", - "title": "Configure GitHub Actions" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#rebasing-new-migrations", - "title": "Rebasing new migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-push", - "title": "Permission denied on db push" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-pull", - "title": "Permission denied on db pull" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#sync-production-project-to-staging", - "title": "Sync production project to staging" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#deploy-a-migration", - "title": "Deploy a migration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#auto-schema-diff", - "title": "Auto schema diff" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#manual-migration", - "title": "Manual migration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#create-a-new-migration", - "title": "Create a new migration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", - "title": "Working with branches" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#managing-branch-environments", - "title": "Managing branch environments" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#subscribing-to-notifications", - "title": "Subscribing to notifications" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#migration-and-seeding-behavior", - "title": "Migration and seeding behavior" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#using-orm-or-custom-seed-scripts", - "title": "Using ORM or custom seed scripts" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#rolling-back-migrations", - "title": "Rolling back migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#seeding-behavior", - "title": "Seeding behavior" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#developing-with-branches", - "title": "Developing with branches" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#local-development-workflow", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#remote-development-workflow", - "title": "Remote development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#switching-between-branches", - "title": "Switching between branches" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#accessing-branch-credentials", - "title": "Accessing branch credentials" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#branch-isolation", - "title": "Branch isolation" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches#next-steps", - "title": "Next steps" + "url": "https://supabase.com/docs/guides/database/secure-data#more-information", + "title": "More information" } ], - "resultChars": 396779 + "resultChars": 88068 } ] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app.json" + "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -6590,34 +6734,67 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-cli-002-declarative-schema", + "eval": "build-functions-005-dual-auth-user-secret", "stage": "build", "product": [ + "edge-functions", + "auth", "database" ], "topic": [ - "declarative-schema", - "migrations" + "sdk", + "rls", + "security" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "schema file updated to include description column", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"error\":\"unauthorized\"}" }, { - "name": "a new migration was generated for the change", - "passed": true + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"5b81009b-6dc6-4a4f-b212-ba200cb5138a\",\"metric\":\"steps_a_mry6flq0\",\"value\":111}]" }, { - "name": "description column exists in the live database", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"5b81009b-6dc6-4a4f-b212-ba200cb5138a\",\"metric\":\"steps_a_mry6flq0\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": false, + "notes": "status 401: {\"error\":\"unauthorized\"}" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"error\":\"unauthorized\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"error\":\"unauthorized\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"error\":\"unauthorized\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -6633,38 +6810,38 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"declarative database schemas supabase local development alter table add column\", limit: 5) { nodes { title href content } } }", + "query": "query {\n searchDocs(query: \"Edge Functions verify JWT apikey service_role request headers supabase js server authenticate user\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href methodName language content }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n }\n totalCount\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/database/tables", - "title": "Tables and Data" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" } ], - "resultChars": 96255 + "resultChars": 46690 } ] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema.json" + "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -6675,379 +6852,419 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-cli-003-pg-cron-queue-workflow", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ - "database", - "edge-functions", - "cron", - "queues" + "storage", + "database" ], "topic": [ - "sql", + "rls", "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "name": "bucket user-files exists", + "passed": true }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "name": "bucket user-files is private", + "passed": true }, { - "name": "process-tasks function drains the queue", + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"cron jobs queue edge function tasks schedule background worker\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { language methodName } } } }", - "hasContent": true, + "notes": "saw: 019f9166-431c-7285-9d30-110739a92ca1/receipt-alpha.pdf, 019f9166-431c-7285-9d30-110739a92ca1/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "The answer creates a private user-files bucket (public=false), keeps RLS in place, and defines authenticated-only SELECT and INSERT policies on storage.objects scoped to bucket_id='user-files' and the user's folder via storage.foldername(name)[1]=auth.uid()::text, with INSERT using WITH CHECK. It also provides supabase-js createSignedUrl code with a 10-minute expiry. No public bucket/getPublicUrl/service-role-client misuse." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage bucket policies authenticated create signed url\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", - "title": "Invoke an Edge Function every minute" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", - "title": "Examples" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", - "title": "Usage " + "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", + "title": "Why can't I upload/list/etc my public bucket?" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", - "title": "Enable the extension" - }, + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + } + ], + "resultChars": 130273 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage RLS policies signed URL upload download user id path\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", - "title": "Signature " + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", - "title": "Usage " + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", - "title": "http_post" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", - "title": "Signature " + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", - "title": "Call an endpoint every minute with pg_cron" - }, + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + } + ], + "resultChars": 104587 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage.objects bucket policy path owner upload download signed URLs\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", - "title": "Execute pg_net in a trigger" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", - "title": "Send multiple table rows in one request" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", - "title": "Limitations" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", - "title": "Resources" + "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", + "title": "Why can't I upload/list/etc my public bucket?" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", - "title": "Debugging requests" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", - "title": "Analyzing responses" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", - "title": "http_get" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", - "title": "Usage " + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", - "title": "http_delete" + "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", + "title": "Smart CDN" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", - "title": "Signature " - }, + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + } + ], + "resultChars": 214760 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"create signed url supabase-js storage\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", - "title": "Inspecting request data" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", - "title": "Inspecting failed requests" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", - "title": "Configuration" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", - "title": "Get current settings" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", - "title": "Alter settings" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", - "title": "Examples" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", - "title": "Invoke a Supabase Edge Function" + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" }, { - "url": "https://supabase.com/docs/guides/functions/background-tasks", - "title": "Background Tasks" + "url": "https://supabase.com/docs/guides/integrations/partner-integration-guide", + "title": "Supabase Partner Integration Guide" }, { - "url": "https://supabase.com/docs/guides/functions/background-tasks#overview", - "title": "Overview" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/functions/background-tasks#handling-errors", - "title": "Handling errors" - }, + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + } + ], + "resultChars": 277974 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage.foldername storage.objects policy path prefix user id\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/background-tasks#testing-background-tasks-locally", - "title": "Testing background tasks locally" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#see-also", - "title": "See also" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-one-request-per-row", - "title": "Why not one request per row?" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#is-10-seconds-a-good-interval-for-processing", - "title": "Is 10 seconds a good interval for processing?" + "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", + "title": "Why can't I upload/list/etc my public bucket?" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-4-create-the-edge-function", - "title": "Step 4: Create the Edge Function" + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-visibility-timeouts-work", - "title": "How do visibility timeouts work?" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-queue-requests-instead-of-processing-them-immediately", - "title": "Why queue requests instead of processing them immediately?" + "url": "https://supabase.com/docs/guides/storage/quickstart", + "title": "Storage Quickstart" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-generate-all-embeddings-in-a-single-edge-function-request", - "title": "Why not generate all embeddings in a single Edge Function request?" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-3-create-queue-and-triggers", - "title": "Step 3: Create queue and triggers" - }, + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + } + ], + "resultChars": 51159 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage.objects owner path policy example foldername\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-2-create-utility-functions", - "title": "Step 2: Create utility functions" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-1-enable-extensions", - "title": "Step 1: Enable extensions" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#implementation", - "title": "Implementation" + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-architecture", - "title": "Understanding the architecture" + "url": "https://supabase.com/docs/guides/storage/quickstart", + "title": "Storage Quickstart" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-challenge", - "title": "Understanding the challenge" + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", + "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#conclusion", - "title": "Conclusion" - }, + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + } + ], + "resultChars": 29223 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-we-handle-retries", - "title": "How do we handle retries?" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#usage", - "title": "Usage" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#1-create-table-to-store-documents-with-embeddings", - "title": "1. Create table to store documents with embeddings" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#2-create-triggers-to-enqueue-embedding-jobs", - "title": "2. Create triggers to enqueue embedding jobs" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#optional-clearing-embeddings-on-update", - "title": "(Optional) Clearing embeddings on update" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#3-insert-and-update-documents", - "title": "3. Insert and update documents" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", - "title": "Inspecting job runs" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", - "title": "Caution: Scheduling system maintenance" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md supabase changelog md breaking-change storage", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase changelog storage bucket RLS breaking change July 2026", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage.from createSignedUrl javascript signedUrl\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", - "title": "Invoke Supabase Edge Function every 30 seconds" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", - "title": "Call a database stored procedure" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", - "title": "Call a database function every 5 minutes" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", - "title": "Run a vacuum every day" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-uploadtosignedurl" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", - "title": "Delete data every week" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#examples", - "title": "Examples" + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", + "title": "Manage Storage Image Transformations usage" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", - "title": "Unschedule a job" + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", - "title": "Activate/Deactivate a job" + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", - "title": "Edit a job" + "url": "https://supabase.com/docs/reference/swift/storage-from-createsignedurl", + "title": "from.createSignedUrl()" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", - "title": "Schedule a job" + "url": "https://supabase.com/docs/reference/swift/storage-from-createsignedurls", + "title": "from.createSignedUrls()" } ], - "resultChars": 142727 + "resultChars": 32315 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"pgmq create queue pop delete read Supabase queues create queue\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"storage create signed url supabase-js expiresIn\", limit: 10) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" }, { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", + "title": "Manage Storage Image Transformations usage" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" }, { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", - "title": "Migrating to Supabase" + "url": "https://supabase.com/docs/guides/integrations/partner-integration-guide", + "title": "Supabase Partner Integration Guide" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" } ], - "resultChars": 50907 + "resultChars": 100131 } ] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -7058,105 +7275,33 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", "product": [ "database" ], "topic": [ - "migrations" + "tests", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", + "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { - "name": "user A cannot force-read user B note", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "status=403" + "notes": "6 passed, 0 failed" }, { - "name": "user B cannot force-read user A note", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "notes": "status=403" + "judgeNotes": "The agent explicitly identified `posts` as the broken tenant isolation policy, stating authenticated members could read all posts because the policy checked any membership rather than membership in the row’s org. It grounded the conclusion in pgTAP testing and reported passing results after applying the fix. It did not blame `notes` or dismiss the tests." } ], "skills": { @@ -7172,491 +7317,303 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Edge Functions get user auth createClient Authorization header anon key\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"pgTAP Supabase tests RLS\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#testing-approaches", + "title": "Testing approaches" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#database-unit-testing-with-pgtap", + "title": "Database unit testing with pgTAP" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 84907 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 503: {\"message\":\"name resolution failed\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 503: {\"message\":\"name resolution failed\"}" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 503: {\"message\":\"name resolution failed\"}" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 503: {\"message\":\"name resolution failed\"}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 503: {\"message\":\"name resolution failed\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 503: {\"message\":\"name resolution failed\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 503: {\"message\":\"name resolution failed\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Edge Functions authentication service_role getUser verify JWT supabase-js\", limit: 5) {\n nodes {\n __typename\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#application-level-testing", + "title": "Application-Level testing" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#test-isolation-strategies", + "title": "Test isolation strategies" }, { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#continuous-integration-testing", + "title": "Continuous integration testing" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#best-practices", + "title": "Best practices" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - } - ], - "resultChars": 84447 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"verify_jwt edge functions auth secret user combined modes withSupabase\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/local-development/testing/overview#real-world-examples", + "title": "Real-World examples" + }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/local-development/testing/overview#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-4", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-3", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#minimize-joins", + "title": "Minimize joins" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#use-security-definer-functions", + "title": "Use security definer functions" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-2", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-filters-to-every-query", + "title": "Add filters to every query" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - } - ], - "resultChars": 120341 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase functions config.toml entrypoint import_map local serve\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-1", + "title": "Benchmarks" + }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#call-functions-with-select", + "title": "Call functions with select" }, { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#row-level-security-in-supabase", + "title": "Row Level Security in Supabase" }, { - "url": "https://supabase.com/docs/guides/functions/dependencies", - "title": "Managing dependencies" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#policies", + "title": "Policies" }, { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#enabling-row-level-security", + "title": "Enabling Row Level Security" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#auto-enable-rls-for-new-tables", + "title": "Auto-enable RLS for new tables" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authenticated-and-unauthenticated-roles", + "title": "Authenticated and unauthenticated roles" }, { - "url": "https://supabase.com/docs/guides/deployment", - "title": "Deployment & Branching" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#creating-policies", + "title": "Creating policies" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#select-policies", + "title": "SELECT policies" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#insert-policies", + "title": "INSERT policies" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-functions-serve", - "title": "Serve all Functions locally" - } - ], - "resultChars": 79104 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019f6c9d-1652-7625-a693-586f70ac5f7c/receipt-alpha.pdf, 019f6c9d-1652-7625-a693-586f70ac5f7c/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Configured a private user-files bucket, owner-scoped authenticated SELECT and INSERT policies with WITH CHECK on storage.objects while keeping RLS intact, and provided supabase-js createSignedUrl code with an expiry. No fail conditions present." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage policy restrict file uploads to authenticated user's folder path RLS bucket storage.objects signed upload signed URL\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href language methodName content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#update-policies", + "title": "UPDATE policies" + }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#delete-policies", + "title": "DELETE policies" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#views", + "title": "Views" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", - "title": "Storage error: 403 Forbidden: 'new row violates row-level security policy' on upload" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#helper-functions", + "title": "Helper functions" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authuid", + "title": "auth.uid()" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 23119 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js storage from createSignedUrl signed url method syntax JavaScript\", limit: 5) { nodes { __typename ... on ClientLibraryFunctionReference { title href language methodName content } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt", + "title": "auth.jwt()" + }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#mfa", + "title": "MFA" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#bypassing-row-level-security", + "title": "Bypassing Row Level Security" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations", + "title": "RLS performance recommendations" }, { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-indexes", + "title": "Add indexes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - } - ], - "resultChars": 47474 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage.objects create policy authenticated foldername auth.uid select insert update delete\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href language methodName content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#more-resources", + "title": "More resources" + }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#specify-roles-in-your-policies", + "title": "Specify roles in your policies" }, { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#4-test-cases", + "title": "4. Test cases:" }, { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" - } - ], - "resultChars": 29933 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "9 passed, 0 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a tenant isolation flaw, specifically exposing posts to authenticated members outside the post's org, and treats pgTAP testing as verification. It does mention other tables, including `memberships` and `notes`, but does not blame `notes` instead of `posts`." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"row level security testing local auth.uid pgtap policies\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#3-rls-policies-declaration", + "title": "3. RLS policies declaration" + }, { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#2-grant-role-privileges", + "title": "2. Grant role privileges" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#1-app-schema-definitions", + "title": "1. App schema definitions" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#testing-focus-areas", + "title": "Testing focus areas" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#what-makes-this-complex", + "title": "What makes this complex?" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#system-overview", + "title": "System overview" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#not-another-todo-app-testing-complex-organizations", + "title": "Not another todo app: Testing complex organizations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#example-advanced-rls-testing", + "title": "Example: Advanced RLS testing" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#benefits", + "title": "Benefits" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#creating-a-pre-test-hook", + "title": "Creating a pre-test hook" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#test-file-organization", + "title": "Test file organization" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#schema-wide-row-level-security-testing", + "title": "Schema-wide Row Level Security testing" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#test-helper-benefits", + "title": "Test helper benefits" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#installing-test-helpers", + "title": "Installing test helpers" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#setting-up-dbdev", + "title": "Setting up dbdev" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended#using-databasedev", + "title": "Using database.dev" }, { "url": "https://supabase.com/docs/guides/database/testing", "title": "Testing Your Database" }, + { + "url": "https://supabase.com/docs/guides/database/testing#running-tests", + "title": "Running tests" + }, + { + "url": "https://supabase.com/docs/guides/database/testing#more-resources", + "title": "More resources" + }, + { + "url": "https://supabase.com/docs/guides/database/testing#writing-tests", + "title": "Writing tests" + }, + { + "url": "https://supabase.com/docs/guides/database/testing#creating-a-test", + "title": "Creating a test" + }, + { + "url": "https://supabase.com/docs/guides/database/testing#testing-using-the-supabase-cli", + "title": "Testing using the Supabase CLI" + }, { "url": "https://supabase.com/docs/guides/database/extensions/pgtap", "title": "pgTAP: Unit Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#overview", + "title": "Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#testing-tables", + "title": "Testing tables" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#testing-columns", + "title": "Testing columns" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#testing-rls-policies", + "title": "Testing RLS policies" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#testing-functions", + "title": "Testing functions" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgtap#resources", + "title": "Resources" } ], - "resultChars": 56902 + "resultChars": 191364 } ] }, @@ -7686,7 +7643,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { "name": "document_sections.embedding is vector(384)", @@ -7696,20 +7653,22 @@ { "name": "HNSW index on the embedding column", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" }, { "name": "index operator class matches the search operator", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" }, { "name": "user A search returns only own sections, best match first", - "passed": true + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" }, { "name": "user B search returns only own sections, best match first", - "passed": true + "passed": false, + "notes": "Could not find the function public.match_document_sections(match_count, query_embedding) in the schema cache" }, { "name": "user A reads only own sections through the API", @@ -7726,136 +7685,72 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search row level security documents owned by user\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"semantic search pgvector row level security documents\", limit: 5) { nodes { title href content ... on Guide { subsections { totalCount } } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", "title": "RAG with Permissions" }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, { "url": "https://supabase.com/docs/guides/ai/semantic-search", "title": "Semantic search" }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, { "url": "https://supabase.com/docs/guides/ai/hybrid-search", "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" } ], - "resultChars": 81105 + "resultChars": 51492 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase vector index hnsw cosine ops syntax extensions.vector\", limit: 10) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"pgvector vector type dimension without size variable length supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/increase-vector-lookup-speeds-by-applying-an-hsnw-index-ohLHUM", - "title": "Increase vector lookup speeds by applying an HSNW index" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", - "title": "IVFFlat indexes" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-indexes", - "title": "Vector indexes" + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" }, { - "url": "https://supabase.com/docs/guides/storage/vector/working-with-indexes", - "title": "Working with Vector Indexes" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { "url": "https://supabase.com/docs/guides/ai/vector-columns", "title": "Vector columns" }, { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" - }, - { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" }, { "url": "https://supabase.com/docs/guides/ai/vecs-python-client", "title": "Python client" } ], - "resultChars": 89986 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Project not specified edge functions endpoint supabase functions host project ref\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/log-drains", - "title": "Log Drains" - }, - { - "url": "https://supabase.com/docs/guides/storage/analytics/connecting-to-analytics-bucket", - "title": "Iceberg Catalog" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - } - ], - "resultChars": 42668 - }, - { - "source": "web_search", - "query": "site:supabase.com \"Project not specified\" \"functions.supabase.co\" supabase", - "pages": [] - }, - { - "source": "web_search", - "query": "\"Project not specified\" \"Supabase\" \"functions\"", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs \"functions/v1\" \"Project not specified\"", - "pages": [] + "resultChars": 36893 } ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions.json" }, { @@ -7885,12 +7780,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails because the Supabase scrape uses basic_auth.password with an injected Secret API key instead of basic_auth.password_file, and docker-compose.yml does not mount that password_file via a volume or Compose secret. The app scrape is preserved and endpoint/HTTPS target are otherwise correct." + "judgeNotes": "Fails because the Supabase scrape uses basic_auth.password instead of basic_auth.password_file, and docker-compose.yml does not mount the password_file via a volume or Compose secret. The app job is preserved and the HTTPS metrics path/target are otherwise present." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README mentions creating a Supabase Secret API key and setting env vars, but it does not instruct placing a matching secret file, lacks a concrete compose restart/reload command, and does not include concrete verification via Prometheus targets, PromQL, Grafana, or equivalent." + "judgeNotes": "README includes the correct Supabase metrics endpoint/auth and Prometheus targets verification, but it does not explain how to create the Supabase Secret API key, does not require placing a matching secret file, and does not mention restarting/reloading the Compose stack after adding the secret. The setup uses .env instead of the required secret file approach." } ], "skills": { @@ -7906,31 +7801,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"metrics prometheus supabase project metrics endpoint observability\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ClientLibraryFunctionReference { title href content } ... on ManagementApiReference { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query {\n searchDocs(query: \"metrics prometheus observability export project metrics\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on ManagementApiReference {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n }\n }\n }\n}", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, { "url": "https://supabase.com/docs/guides/telemetry/metrics", "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, + { + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, { "url": "https://supabase.com/docs/guides/platform/read-replicas", "title": "Read Replicas" } ], - "resultChars": 32819 + "resultChars": 32694 } ] }, @@ -7992,31 +7887,99 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions secrets deploy WEATHER_API_KEY runtime environment variables\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy supabase functions secrets set\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } totalCount } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" + } + ], + "resultChars": 39297 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions deploy secret environment variable verify runtime Deno.env.get\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + } + ], + "resultChars": 34518 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions verify_jwt false public browser access apikey header unauthorized\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" } ], - "resultChars": 51647 + "resultChars": 94959 } ] }, @@ -8078,935 +8041,1514 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting docker compose Supabase official docker compose env secrets\", limit: 5) { nodes { title href ... on Guide { content subsections { nodes { title href content } } } ... on CLICommandReference { content } ... on TroubleshootingGuide { content } } totalCount } }", + "query": "query {\n searchDocs(query: \"self-hosted Supabase docker compose API_EXTERNAL_URL Envoy default gateway docker setup\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n ... on ClientLibraryFunctionReference { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", - "title": "How it works" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", - "title": "What client SDK sends" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", - "title": "Kong API gateway routing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", - "title": "Differences from the Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", - "title": "Backward compatibility" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", - "title": "Rotating the new API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", - "title": "Regenerating asymmetric key pair" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { "url": "https://supabase.com/docs/guides/self-hosting/docker", "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", - "title": "Setting database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", - "title": "Configuring phone login, SMS, and MFA" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" + } + ], + "resultChars": 105681 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Self-Hosting with Docker API_EXTERNAL_URL env file docker-compose.yml .env.example self-hosted auth keys sb_publishable sb_secret\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", - "title": "Accessing Postgres through Supavisor" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", - "title": "Contents" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", - "title": "System requirements" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", - "title": "Installing Supabase" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", + "title": "Configure Phone Login & MFA" + } + ], + "resultChars": 216571 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Self-Hosting with Docker Supabase docker-compose.yml .env.example volumes/api/envoy docker-compose.caddy.yml docker-compose.nginx.yml\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", - "title": "Quick start (Linux)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", - "title": "Manual installation" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", - "title": "Configuring and securing Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", - "title": "Generate keys and secrets" + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", - "title": "Configure Supabase URLs" - }, + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + } + ], + "resultChars": 74094 + } + ] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "created auth sessions", + "passed": false, + "notes": "Internal server error" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user revoke sessions auth.users delete_account\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", - "title": "Where to find your credentials" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", - "title": "Studio authentication" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", - "title": "Starting and stopping" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", - "title": "Accessing Supabase Studio (Dashboard)" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", - "title": "Accessing Postgres" - }, + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" + } + ], + "resultChars": 41595 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"publishable secret key anon service_role frontend RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", - "title": "Accessing Edge Functions" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", - "title": "Accessing APIs" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", - "title": "Enabling analytics" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", - "title": "Configuring HTTPS" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", - "title": "Managing the stack" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", - "title": "Updating" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", - "title": "Uninstalling" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", - "title": "Advanced topics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#get-started", - "title": "Get started" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#community-driven-projects", - "title": "Community-driven projects" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting#about-self-hosting", - "title": "About self-hosting" - }, + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 91173 + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication, added only public.orders via ALTER PUBLICATION, and preserved courier_locations, RLS, and policies." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query:\"Supabase Realtime postgres_changes publication table wal_level replication\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting#how-self-hosted-supabase-differs", - "title": "How self-hosted Supabase differs" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/self-hosting#your-responsibilities-when-self-hosting", - "title": "Your responsibilities when self-hosting" + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" }, { - "url": "https://supabase.com/docs/guides/self-hosting#telemetry", - "title": "Telemetry" + "url": "https://supabase.com/docs/guides/database/replication/pipelines", + "title": "Set up Pipelines" }, { - "url": "https://supabase.com/docs/guides/self-hosting#support-and-community", - "title": "Support and community" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/self-hosting#enterprise-self-hosting", - "title": "Enterprise self-hosting" - }, + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + } + ], + "resultChars": 90112 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query:\"Realtime postgres_changes publication table not receiving events orders\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-the-aws-cli", - "title": "Test with the AWS CLI" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#how-to-configure-an-s3-backend", - "title": "How to configure an S3 backend" + "url": "https://supabase.com/docs/guides/realtime/protocol", + "title": "Realtime Protocol" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-rustfs", - "title": "Using RustFS" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-minio", - "title": "Using MinIO" - }, + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" + } + ], + "resultChars": 124600 + } + ] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as the affected function and described recurring intermittent HTTP 503s from roughly 07:00 through 12:00 UTC on 2026-04-28, while distinguishing unrelated/one-off errors." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": true, + "judgeNotes": "Attributes recurring image-transform 503s to the API/gateway/platform layer before the function, grounded in no matching edge-function execution logs and successful nearby 200 executions. It distinguishes avatar-upload's 500 as separate function-level/server-side issue." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended concrete next steps including checking deploys/rollbacks/health-check failures, pulling deeper edge runtime/platform logs, inspecting a specific function path, and documenting the affected incident window." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API behavior; kept RLS enabled; created authenticated SELECT policy scoped to user_id = auth.uid() and authenticated INSERT policy with WITH CHECK enforcing user ownership." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Exposing a Table to the Data API grants anon authenticated RLS\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-aws-s3", - "title": "Using AWS S3" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#s3-compatible-providers", - "title": "S3-compatible providers" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#verify", - "title": "Verify" + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0023_sensitive_columns_exposed", + "title": "Database Advisor: Lint 0023_sensitive_columns_exposed" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#session-token", - "title": "Session token" + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0017_foreign_table_in_api", + "title": "Database Advisor: Lint 0017_foreign_table_in_api" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#signature-mismatch-errors", - "title": "Signature mismatch errors" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#enable-the-s3-protocol-endpoint", - "title": "Enable the S3 protocol endpoint" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#tus-upload-errors-on-cloudflare-r2", - "title": "TUS upload errors on Cloudflare R2" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#permission-denied-on-uploads", - "title": "Permission denied on uploads" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#upload-urls-point-to-localhost", - "title": "Upload URLs point to localhost" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-rclone", - "title": "Test with rclone" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#nonce-check-failure-on-mobile-google-sign-in", - "title": "Nonce check failure on mobile (Google Sign In)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#oauth-request-flow", - "title": "OAuth request flow" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-environment-variables", - "title": "Auth environment variables" - }, + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0027_pg_graphql_authenticated_table_exposed", + "title": "Database Advisor: Lint 0027_pg_graphql_authenticated_table_exposed" + } + ], + "resultChars": 65012 + } + ] + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true + }, + { + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true + }, + { + "name": "remote migration history matches local migration files", + "passed": false, + "notes": "remote-only versions still unreconciled: [\"20240115000000\"] (remote: [\"20240101000000\",\"20240115000000\",\"20240220000000\"], local: [\"20240101000000\",\"20240220000000\"])" + }, + { + "name": "local migrations are a valid reconciled sequence", + "passed": false, + "notes": "expected 20240101000000_create_profiles → → 20240220000000_add_avatar_url, strictly ascending; got [20240101000000_create_profiles.sql, 20240220000000_add_avatar_url.sql]" + }, + { + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": false, + "judgeNotes": "FAIL: The avatar_url migration was applied via direct psql SQL in command #25 (`alter table public.profiles add column...`), not via `supabase db push`. The migration history was also edited directly in #25 with an `insert into supabase_migrations.schema_migrations...`, which is an explicit disallowed workaround. No successful `supabase db push` occurred." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"composite index order by desc recent user events query optimization\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-by-step-configuration", - "title": "Step-by-step configuration" + "url": "https://supabase.com/docs/guides/storage/analytics/query-with-postgres", + "title": "Query with Postgres" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-1-register-your-app-with-the-provider", - "title": "Step 1: Register your app with the provider" - }, + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + } + ], + "resultChars": 8366 + } + ] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"RLS policy auth.uid org_id membership workspace row level security\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on ClientLibraryFunctionReference { title href methodName language content href }\n ... on CLICommandReference { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-2-configure-environment-variables", - "title": "Step 2: Configure environment variables" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-3-enable-the-matching-lines-in-docker-compose-configuration", - "title": "Step 3: Enable the matching lines in Docker Compose configuration" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-4-restart-the-auth-service", - "title": "Step 4: Restart the auth service" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-5-verify-the-configuration", - "title": "Step 5: Verify the configuration" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-specific-setup", - "title": "Provider-specific setup" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + } + ], + "resultChars": 77495 + } + ] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": false, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": false + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": false, + "notes": "found 1 migration file(s)" + }, + { + "name": "description column exists in the live database", + "passed": false + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-002-declarative-schema.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 1 -> 2" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 5) from the queue" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"pg_cron schedule cron.job unschedule enqueue pgmq send pop delete public wrapper\", limit: 5) {\n nodes {\n title\n href\n content\n ... on Guide {\n subsections {\n totalCount\n nodes {\n title\n href\n content\n }\n }\n }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview", + "title": "Foreign Data Wrappers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#other-supported-providers", - "title": "Other supported providers" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-service-fails-to-start", - "title": "Auth service fails to start" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#remote-servers", + "title": "Remote servers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#environment-variable-reference", - "title": "Environment variable reference" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#foreign-tables", + "title": "Foreign tables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#etl-with-wrappers", + "title": "ETL with Wrappers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#variables-added-to-the-environment-but-provider-still-not-working", - "title": "Variables added to the environment but provider still not working" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#on-demand-etl-with-wrappers", + "title": "On-demand ETL with Wrappers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#test-the-login-flow", - "title": "Test the login flow" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#batch-etl-with-wrappers", + "title": "Batch ETL with Wrappers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#webassembly-wrappers", + "title": "WebAssembly Wrappers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#site-url-or-redirect-url-errors-after-login", - "title": "Site URL or redirect URL errors after login" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#security", + "title": "Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-not-enabled-or-provider-seen-as-false-in-settings" - } - ], - "resultChars": 341620 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Self-Hosting with Docker quick start Linux generate keys env example docker compose\", limit: 5) { nodes { title href ... on Guide { content subsections { nodes { title href content } } } } totalCount } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#resources", + "title": "Resources" + }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", - "title": "Enabling analytics" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", + "title": "Enable the extension" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", + "title": "Configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", - "title": "Accessing Edge Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", + "title": "Inspecting failed requests" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", - "title": "Accessing APIs" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", + "title": "Inspecting request data" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", - "title": "Configuring phone login, SMS, and MFA" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", - "title": "Configuring HTTPS" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", + "title": "http_delete" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", - "title": "Managing the stack" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", - "title": "Updating" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", + "title": "http_post" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", - "title": "Uninstalling" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", - "title": "Advanced topics" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", + "title": "Signature " }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", + "title": "http_get" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", - "title": "Setting database password" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", + "title": "Resources" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", + "title": "Limitations" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", + "title": "Send multiple table rows in one request" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", + "title": "Analyzing responses" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", + "title": "Debugging requests" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", + "title": "Execute pg_net in a trigger" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", + "title": "Call an endpoint every minute with pg_cron" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", + "title": "Invoke a Supabase Edge Function" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", - "title": "Accessing Postgres through Supavisor" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", + "title": "Examples" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", + "title": "Alter settings" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", - "title": "Contents" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", + "title": "Get current settings" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", - "title": "System requirements" + "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", + "title": "How does Cron work?" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", - "title": "Installing Supabase" + "url": "https://supabase.com/docs/guides/cron#resources", + "title": "Resources" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", - "title": "Quick start (Linux)" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", - "title": "Manual installation" + "url": "https://supabase.com/docs/guides/platform/upgrading#upgrading-to-pg_graphql-160", + "title": "Upgrading to pg_graphql 1.6.0" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", - "title": "Configuring and securing Supabase" + "url": "https://supabase.com/docs/guides/platform/upgrading#upgrading-to-postgres-17", + "title": "Upgrading to Postgres 17" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", - "title": "Generate keys and secrets" + "url": "https://supabase.com/docs/guides/platform/upgrading#specific-upgrade-notes", + "title": "Specific upgrade notes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", - "title": "Configure Supabase URLs" + "url": "https://supabase.com/docs/guides/platform/upgrading#post-upgrade-validation", + "title": "Post-upgrade validation" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", - "title": "Where to find your credentials" + "url": "https://supabase.com/docs/guides/platform/upgrading#database-size-reduction", + "title": "Database size reduction" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", - "title": "Studio authentication" + "url": "https://supabase.com/docs/guides/platform/upgrading#authentication-method-changes---deprecating-md5-in-favor-of-scram-sha-256", + "title": "Authentication method changes - deprecating md5 in favor of scram-sha-256" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", - "title": "Starting and stopping" + "url": "https://supabase.com/docs/guides/platform/upgrading#extensions", + "title": "Extensions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", - "title": "Accessing Supabase Studio (Dashboard)" + "url": "https://supabase.com/docs/guides/platform/upgrading#pg_cron-records", + "title": "pg_cron records" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", - "title": "Accessing Postgres" + "url": "https://supabase.com/docs/guides/platform/upgrading#objects-dependent-on-postgres-extensions", + "title": "Objects dependent on Postgres extensions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/platform/upgrading#disk-sizing", + "title": "Disk sizing" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" + "url": "https://supabase.com/docs/guides/platform/upgrading#time-limits", + "title": "Time limits" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", - "title": "What client SDK sends" + "url": "https://supabase.com/docs/guides/platform/upgrading#breaking-changes", + "title": "Breaking changes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", - "title": "How it works" + "url": "https://supabase.com/docs/guides/platform/upgrading#logical-replication", + "title": "Logical replication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", - "title": "Regenerating asymmetric key pair" + "url": "https://supabase.com/docs/guides/platform/upgrading#caveats", + "title": "Caveats" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", - "title": "Rotating the new API keys" + "url": "https://supabase.com/docs/guides/platform/upgrading#pause-and-restore", + "title": "Pause and restore" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", - "title": "Backward compatibility" - }, + "url": "https://supabase.com/docs/guides/platform/upgrading#in-place-upgrades", + "title": "In-place upgrades" + } + ], + "resultChars": 88038 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"pgmq pop send delete queue create message_record\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", - "title": "Differences from the Supabase platform" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" - }, + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + } + ], + "resultChars": 32735 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=403" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=403" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY/PUBLISHABLE_KEY — new API keys are required for @supabase/server; is the stack running on a new-enough CLI? got keys: ANON_KEY, DB_URL, JWT_SECRET, PUBLISHABLE_KEY, SECRET_KEY, SERVICE_ROLE_KEY" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"edge function service role key request header apikey env variable user access token supabase-js @supabase/functions\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n }\n totalCount\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", - "title": "Kong API gateway routing" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", - "title": "Additional resources" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 52103 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"@supabase/server userClaims id sub authMode ctx.userClaims type\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n totalCount\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-4-restart-the-auth-service", - "title": "Step 4: Restart the auth service" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#environment-variable-reference", - "title": "Environment variable reference" + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-service-fails-to-start", - "title": "Auth service fails to start" + "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#nonce-check-failure-on-mobile-google-sign-in", - "title": "Nonce check failure on mobile (Google Sign In)" + "url": "https://supabase.com/docs/reference/swift/auth-getclaims", + "title": "getClaims()" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#site-url-or-redirect-url-errors-after-login", - "title": "Site URL or redirect URL errors after login" + "url": "https://supabase.com/docs/guides/auth/auth-hooks/custom-access-token-hook", + "title": "Custom Access Token Hook" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#variables-added-to-the-environment-but-provider-still-not-working", - "title": "Variables added to the environment but provider still not working" - }, + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + } + ], + "resultChars": 173826 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"functions config.toml verify_jwt function section supabase config.toml [functions.my-function]\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n }\n totalCount\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#oauth-request-flow", - "title": "OAuth request flow" + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-environment-variables", - "title": "Auth environment variables" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-by-step-configuration", - "title": "Step-by-step configuration" + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-1-register-your-app-with-the-provider", - "title": "Step 1: Register your app with the provider" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-2-configure-environment-variables", - "title": "Step 2: Configure environment variables" + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-3-enable-the-matching-lines-in-docker-compose-configuration", - "title": "Step 3: Enable the matching lines in Docker Compose configuration" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/functions/examples/sentry-monitoring", + "title": "Monitoring with Sentry" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-5-verify-the-configuration", - "title": "Step 5: Verify the configuration" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-specific-setup", - "title": "Provider-specific setup" - }, + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + } + ], + "resultChars": 64170 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"functions main index.ts edge runtime bootstrap entrypoint self-hosted functions main worker\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n }\n totalCount\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#other-supported-providers", - "title": "Other supported providers" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#test-the-login-flow", - "title": "Test the login flow" + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", - "title": "Changes to function code not reflected after editing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", - "title": "Copying functions from Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", - "title": "Deploying functions to a remote server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", - "title": "Managing functions via dashboard" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", - "title": "Internal vs external URLs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", - "title": "500 error on invocation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", - "title": "Calling Supabase services from functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", - "title": "Accessing variables in functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", - "title": "Using inline environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", - "title": "Using an env file (recommended)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", - "title": "Custom environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", - "title": "Step 3: Invoke your function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", - "title": "Step 2: Restart the functions service to pick up the new function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", - "title": "Step 1: Add a new function directory and the function code" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", - "title": "Create a new function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", - "title": "Invoke the default function" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", - "title": "Memory or timeout errors" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", - "title": "Custom env vars not available in functions" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-not-enabled-or-provider-seen-as-false-in-settings" + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#401-invalid-jwt" + "url": "https://supabase.com/docs/guides/ai/quickstarts/generate-text-embeddings", + "title": "Generate Embeddings" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" } ], - "resultChars": 323616 + "resultChars": 108132 } ] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "build-storage-001-private-bucket-access", + "stage": "build", "product": [ - "auth" + "storage", + "database" ], "topic": [ - "security", + "rls", "sdk" ], "suite": "benchmark", @@ -9014,1441 +9556,1353 @@ "passed": true, "checks": [ { - "name": "victim session active before delete-account", + "name": "bucket user-files exists", "passed": true }, { - "name": "delete_account flow ran for the victim", + "name": "bucket user-files is private", "passed": true }, { - "name": "delete-account revokes the user's sessions", + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", "passed": true, - "notes": "sessions left: 0" + "notes": "saw: 019f9165-9d93-775b-9383-1a9758ea3d2c/receipt-alpha.pdf, 019f9165-9d93-775b-9383-1a9758ea3d2c/receipt-beta.pdf" }, { - "name": "deleted user's refresh token is rejected", + "name": "user B cannot read user A files", "passed": true }, { - "name": "deleted user cannot sign back in", + "name": "anon reads no files", "passed": true }, { - "name": "other users keep their sessions and access", + "name": "user A can upload into own folder", "passed": true }, { - "name": "diagnosed and explained session revocation", + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets rubric: identifies soft-delete-only delete_account as cause, changes flow to delete auth.users and closes RLS data path for stale JWTs, explains JWT expiry caveat consistently, and correctly distinguishes publishable vs secret keys." + "judgeNotes": "Meets rubric: private user-files bucket, authenticated owner-scoped storage.objects SELECT and INSERT policies with RLS retained, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md supabase changelog md", - "pages": [] - }, - { - "source": "web_search", - "query": "Supabase changelog breaking change auth delete user sessions publishable secret key", - "pages": [] - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user revoke sessions auth.admin.deleteUser auth.sessions\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query {\n searchDocs(query: \"storage policies authenticated users own folder signed upload select bucket create bucket signed urls supabase-js\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data#accessing-user-data-via-api", - "title": "Accessing user data via API" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data#adding-and-retrieving-user-metadata", - "title": "Adding and retrieving user metadata" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data#exporting-users", - "title": "Exporting users" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data#deleting-users", - "title": "Deleting users" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#row-level-security-policies-rls", - "title": "Row level security policies (RLS)" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#access-token", - "title": "Access token" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#authenticator-app", - "title": "Authenticator app" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#authorization", - "title": "Authorization" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#identity-provider", - "title": "Identity provider" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#json-web-token-jwt", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#jwt-signing-secret", - "title": "JWT signing secret" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#multi-factor-authentication-mfa-or-2fa", - "title": "Multi-factor authentication (MFA or 2FA)" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#nonce", - "title": "Nonce" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#oauth", - "title": "OAuth" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#oidc", - "title": "OIDC" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#one-time-password-otp", - "title": "One-time password (OTP)" + "url": "https://supabase.com/docs/guides/storage", + "title": "Storage" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#password-hashing-function", - "title": "Password hashing function" + "url": "https://supabase.com/docs/guides/storage/s3/compatibility", + "title": "S3 Compatibility" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#password-strength", - "title": "Password strength" - }, + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/firebase-storage", + "title": "Migrated from Firebase Storage to Supabase" + } + ], + "resultChars": 92209 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"storage access control policy examples folder prefix auth.uid objects name like storage.objects\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/resources/glossary#pkce", - "title": "PKCE" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#provider-refresh-token", - "title": "Provider refresh token" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#provider-tokens", - "title": "Provider tokens" + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#refresh-token", - "title": "Refresh token" + "url": "https://supabase.com/docs/guides/storage/quickstart", + "title": "Storage Quickstart" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#refresh-token-flow", - "title": "Refresh token flow" + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#replay-attack", - "title": "Replay attack" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#saml", - "title": "SAML" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#session", - "title": "Session" + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", + "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#single-sign-on-sso", - "title": "Single-sign on (SSO)" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" }, { - "url": "https://supabase.com/docs/guides/resources/glossary#time-based-one-time-password-totp", - "title": "Time-based one-time password (TOTP)" + "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", + "title": "Why can't I upload/list/etc my public bucket?" } ], - "resultChars": 35929 + "resultChars": 31315 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"migrating to new API keys publishable secret frontend RLS\", limit: 5) { nodes { __typename title href content } } }", + "query": "query {\n searchDocs(query: \"storage.foldername helper function storage.folder name first segment\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 109403 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"revoke sessions delete user auth.sessions sign out current user delete account\", limit: 10) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signout", - "title": "signOut()" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" } ], - "resultChars": 84801 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs deleting a user does not automatically sign out a user JWT remain valid until it has expired Supabase", - "pages": [] + "resultChars": 3857 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", "product": [ - "realtime", "database" ], "topic": [ - "sdk" + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "4 passed, 0 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as the tenant isolation flaw, specifically that the policy only checked for any membership rather than matching `posts.org_id`, and reports pgTAP validation as authoritative. It does not blame `notes` or dismiss the test results." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", - "passed": true + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" }, { - "name": "courier_locations still in supabase_realtime publication", - "passed": true + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "publication still publishes INSERT events", + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "user B search returns only own sections, best match first", "passed": true }, { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "name": "user A reads only own sections through the API", + "passed": true }, { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The assistant correctly identified that subscriptions could reach SUBSCRIBED while INSERT events for orders were silent because public.orders was missing from the supabase_realtime publication. It applied the narrow fix `alter publication supabase_realtime add table public.orders;`, verified both orders and courier_locations remained in the existing publication, and did not alter RLS/policies or recreate/drop the publication." + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase_realtime publication postgres_changes table not receiving events\", limit: 5) {\n nodes {\n title\n href\n content\n ... on Guide {\n subsections {\n nodes {\n title\n href\n content\n }\n }\n }\n }\n }\n}", + "query": "query { searchDocs(query: \"Supabase row level security policies auth.uid documents owner_id\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-inserts", - "title": "Streaming inserts" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-broadcast", - "title": "Using Broadcast" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#broadcast-authorization", - "title": "Broadcast authorization" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger-function", - "title": "Create a trigger function" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger", - "title": "Create a trigger" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#listening-on-client-side", - "title": "Listening on client side" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-postgres-changes", - "title": "Using Postgres Changes" - }, + "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", + "title": "Firebase Auth" + } + ], + "resultChars": 55638 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pgvector Supabase create extension vector hnsw match_document_sections RLS auth.uid policies\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#enable-postgres-changes", - "title": "Enable Postgres Changes" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-updates", - "title": "Streaming updates" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-lt", - "title": "Less than (lt)" + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#quick-start", - "title": "Quick start" - }, + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + } + ], + "resultChars": 61684 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": false, + "judgeNotes": "Fails: Supabase scrape uses basic_auth.password instead of password_file, and docker-compose.yml does not mount the password_file via a volume or Compose secret. App scrape is preserved and endpoint/target are otherwise correct." + }, + { + "name": "documented live deployment and verification steps", + "passed": false, + "judgeNotes": "README includes Secret API key creation, Compose restart, and Prometheus target verification, but it does not require placing the matching secret file; it uses observability/.env instead. This fails the required secret setup criterion." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs prometheus metrics supabase project observability", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#usage", - "title": "Usage" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-schemas", - "title": "Listening to specific schemas" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-insert-events", - "title": "Listening to INSERT events" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-update-events", - "title": "Listening to UPDATE events" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-delete-events", - "title": "Listening to DELETE events" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-tables", - "title": "Listening to specific tables" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-multiple-changes", - "title": "Listening to multiple changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#filtering-for-specific-changes", - "title": "Filtering for specific changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#available-filters", - "title": "Available filters" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#equal-to-eq", - "title": "Equal to (eq)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#not-equal-to-neq", - "title": "Not equal to (neq)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-or-equal-to-lte", - "title": "Less than or equal to (lte)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-gt", - "title": "Greater than (gt)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-or-equal-to-gte", - "title": "Greater than or equal to (gte)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#contained-in-list-in", - "title": "Contained in list (in)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#receiving-old-records", - "title": "Receiving old records" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#private-schemas", - "title": "Private schemas" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#custom-tokens", - "title": "Custom tokens" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#refreshed-tokens", - "title": "Refreshed tokens" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#limitations", - "title": "Limitations" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#delete-events-are-not-filterable", - "title": "Delete events are not filterable" - }, + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted" + } + ] + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Edge Function URL project ref functions.supabase.co functions/v1\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#spaces-in-table-names", - "title": "Spaces in table names" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#database-instance-and-realtime-performance", - "title": "Database instance and realtime performance" + "url": "https://supabase.com/docs/guides/functions/storage-caching", + "title": "Integrating with Supabase Storage" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/storage/analytics/connecting-to-analytics-bucket", + "title": "Iceberg Catalog" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#methodology", - "title": "Methodology" + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#workloads", - "title": "Workloads" - }, + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + } + ], + "resultChars": 35102 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"functions/v1 invoke curl project-ref supabase.co/functions/v1\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#results", - "title": "Results" + "url": "https://supabase.com/docs/guides/api/quickstart", + "title": "Build an API route in less than 2 minutes." }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-using-websockets", - "title": "Broadcast: Using WebSockets" + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-using-the-database", - "title": "Broadcast: Using the database" + "url": "https://supabase.com/docs/reference/cli/supabase-projects-delete", + "title": "Delete a Supabase project" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-impact-of-payload-size", - "title": "Broadcast: Impact of payload size" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#1kb-payload", - "title": "1KB payload" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#10kb-payload", - "title": "10KB payload" + "url": "https://supabase.com/docs/reference/cli/supabase-functions-deploy", + "title": "Deploy a Function to Supabase" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#50kb-payload", - "title": "50KB payload" - }, + "url": "https://supabase.com/docs/reference/cli/supabase-projects-list", + "title": "List all Supabase projects" + } + ], + "resultChars": 45778 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Edge Function invoke endpoint management API invoke function\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-scalability-scenarios", - "title": "Broadcast: Scalability scenarios" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#realtime-auth", - "title": "Realtime Auth" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence-errors", - "title": "Presence errors" + "url": "https://supabase.com/docs/guides/ai-tools/mcp", + "title": "Supabase MCP Server" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#access-token-refresh", - "title": "Access token refresh" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#join-errors", - "title": "Join errors" + "url": "https://supabase.com/docs/reference/cli/supabase-functions", + "title": "Manage Supabase Edge functions" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#websocket-connection-setup", - "title": "WebSocket connection setup" + "url": "https://supabase.com/docs/guides/functions/ai-models", + "title": "Running AI Models" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#protocol-messages", - "title": "Protocol messages" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#100", - "title": "1.0.0" - }, + "url": "https://supabase.com/docs/guides/telemetry/reports", + "title": "Reports" + } + ], + "resultChars": 149428 + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"self-hosted docker compose supabase secrets JWT anon service_role postgresql kong studio\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/protocol#200", - "title": "2.0.0" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#text-frames", - "title": "Text frames" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#binary-frames", - "title": "Binary frames" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#user-broadcast-push", - "title": "User Broadcast Push" + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove superuser access from Studio" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#user-broadcast", - "title": "User Broadcast" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#event-types", - "title": "Event types" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#client-sent-events", - "title": "Client sent events" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_join", - "title": "phx_join" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_leave", - "title": "phx_leave" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#heartbeat", - "title": "heartbeat" - }, + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + } + ], + "resultChars": 176548 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Self-Hosting with Docker quick start Linux generate-keys.sh .env.example docker-compose.yml kong.yml kong-entrypoint.sh volumes/api\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/protocol#access_token", - "title": "access_token" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-text-frame", - "title": "broadcast (text frame)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-binary-frame", - "title": "broadcast (binary frame)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence", - "title": "presence" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#server-sent-events", - "title": "Server sent events" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_close", - "title": "phx_close" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_error", - "title": "phx_error" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", + "title": "Custom Email Templates" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_reply", - "title": "phx_reply" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#system", - "title": "system" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-text-frame-1", - "title": "broadcast (text frame)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-binary-frame-1", - "title": "broadcast (binary frame)" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#postgres_changes", - "title": "postgres_changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence_state", - "title": "presence_state" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence_diff", - "title": "presence_diff" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#reconnection", - "title": "Reconnection" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#channel-level-system-errors", - "title": "Channel-level system errors" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#error-handling", - "title": "Error handling" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#postgres-changes-subscription-errors", - "title": "Postgres Changes subscription errors" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-errors", - "title": "Broadcast errors" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#next-steps", - "title": "Next steps" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#framework-examples", - "title": "Framework examples" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#integration-guides", - "title": "Integration guides" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#advanced-topics", - "title": "Advanced topics" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#security--configuration", - "title": "Security & configuration" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#core-features", - "title": "Core features" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-postgres-changes", - "title": "When to use Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-presence", - "title": "When to use Presence" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-broadcast", - "title": "When to use Broadcast" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#choose-the-right-feature", - "title": "Choose the right feature" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#clean-up-subscriptions", - "title": "Clean up subscriptions" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#follow-naming-conventions", - "title": "Follow naming conventions" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#use-private-channels", - "title": "Use private channels" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#essential-best-practices", - "title": "Essential best practices" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#53-using-database-triggers", - "title": "5.3 using database triggers" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#52-using-httprest-api", - "title": "5.2 using HTTP/REST API" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#51-using-client-libraries", - "title": "5.1 using client libraries" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#5-send-and-receive-messages", - "title": "5. Send and receive messages" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#4-set-up-authorization", - "title": "4. Set up authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#3-create-your-first-channel", - "title": "3. Create your first Channel" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#get-api-details", - "title": "Get API details" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#2-initialize-the-client", - "title": "2. Initialize the client" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#1-install-the-client-library", - "title": "1. Install the client library" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#quick-start", - "title": "Quick start" + "url": "https://supabase.com/docs/guides/self-hosting/enable-mcp", + "title": "Enabling MCP Server Access" } ], - "resultChars": 335915 + "resultChars": 132233 + }, + { + "source": "web_search", + "query": "site:github.com supabase/supabase docker docker-compose.yml volumes/api/kong.yml generate-keys.sh master", + "pages": [] } ] }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "eval": "investigate-auth-001-deleted-user-access", "stage": "investigate", "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant named `image-transform` as the affected function and explicitly identified the recurring 503 pattern across 2026-04-28 from 07:00Z to 12:00Z, listing all 8 gateway failures. It did not misattribute the issue to old billing-webhook errors." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes recurring image-transform 503s to gateway/Functions-layer instability rather than function code, and grounds this in gateway 503 pattern, successful 200s between failures on same deployed version, and distinction from avatar-upload's isolated function-level 500." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including checking for a platform incident/rollout during the affected time window, adding explicit function logging, adding retry/backoff, reviewing dependency changes, and collecting exact request IDs to distinguish gateway from handler failures." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", "auth" ], "topic": [ - "rls", + "security", "sdk" ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { - "name": "RLS still enabled on bookmarks", - "passed": true + "name": "victim session active before delete-account", + "passed": false, + "notes": "permission denied for table users" }, { - "name": "user A reads own bookmarks", + "name": "delete_account flow ran for the victim", "passed": true }, { - "name": "user B cannot read user A bookmarks", - "passed": true + "name": "delete-account revokes the user's sessions", + "passed": false, + "notes": "sessions left: 1" }, { - "name": "anon reads no bookmarks", - "passed": true + "name": "deleted user's refresh token is rejected", + "passed": false, + "notes": "refresh token still produces a session" }, { - "name": "user A can save a new bookmark", - "passed": true + "name": "deleted user cannot sign back in", + "passed": false, + "notes": "deleted account can still sign in" }, { - "name": "user B cannot insert a bookmark as user A", - "passed": true + "name": "other users keep their sessions and access", + "passed": false, + "notes": "permission denied for table users" }, { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all/zero Data API rows, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT using WITH CHECK. Extra index and verification are acceptable." + "name": "diagnosed and explained session revocation", + "passed": false, + "judgeNotes": "Fails because the implemented delete flow only sets auth.users.deleted_at and does not actually delete the auth user or remove identities/sessions/refresh tokens, so it does not demonstrate real auth/session revocation as required. The diagnosis, RLS mitigation for stale JWT Data API access, and publishable-vs-secret key explanation are mostly correct, but the core revocation fix is insufficient." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Exposing a Table to the Data API RLS grants anon authenticated bookmarks\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Supabase delete user revoke sessions auth.users delete account\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api#enforce-additional-rules-on-each-request", - "title": "Enforce additional rules on each request" + "url": "https://supabase.com/docs/guides/auth/managing-user-data#deleting-users", + "title": "Deleting users" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api#add-rls-policies", - "title": "Add RLS policies" + "url": "https://supabase.com/docs/guides/auth/managing-user-data#accessing-user-data-via-api", + "title": "Accessing user data via API" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api#disable-the-data-api", - "title": "Disable the Data API" + "url": "https://supabase.com/docs/guides/auth/managing-user-data#adding-and-retrieving-user-metadata", + "title": "Adding and retrieving user metadata" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api#examples", - "title": "Examples" + "url": "https://supabase.com/docs/guides/auth/managing-user-data#exporting-users", + "title": "Exporting users" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api#accessing-request-information", - "title": "Accessing request information" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api#grant-access-explicitly", - "title": "Grant access explicitly" + "url": "https://supabase.com/docs/guides/auth/sessions#frequently-asked-questions", + "title": "Frequently asked questions" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api#default-privileges-for-new-tables-and-functions", - "title": "Default privileges for new tables and functions" + "url": "https://supabase.com/docs/guides/auth/sessions#limiting-session-lifetime-and-number-of-allowed-sessions-per-user", + "title": "Limiting session lifetime and number of allowed sessions per user" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api#use-a-dedicated-api-schema", - "title": "Use a dedicated API schema" + "url": "https://supabase.com/docs/guides/auth/sessions#what-is-a-session", + "title": "What is a session?" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/auth/sessions#access-token-jwt-claims", + "title": "Access token (JWT) claims" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#more-information", - "title": "More information" + "url": "https://supabase.com/docs/guides/auth/sessions#initiating-a-session", + "title": "Initiating a session" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#frontend-access", - "title": "Frontend access" + "url": "https://supabase.com/docs/guides/auth/sessions#what-are-the-benefits-of-using-access-and-refresh-tokens-instead-of-traditional-sessions", + "title": "What are the benefits of using access and refresh tokens instead of traditional sessions?" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#direct-database-connections", - "title": "Direct database connections" + "url": "https://supabase.com/docs/guides/auth/sessions#what-is-refresh-token-reuse-detection-and-what-does-it-protect-from", + "title": "What is refresh token reuse detection and what does it protect from?" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#edge-functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/auth/sessions#what-are-recommended-values-for-access-token-jwt-expiration", + "title": "What are recommended values for access token (JWT) expiration?" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#data-api", - "title": "Data API" + "url": "https://supabase.com/docs/guides/auth/sessions#using-http-only-cookies-to-store-access-and-refresh-tokens", + "title": "Using HTTP-only cookies to store access and refresh tokens" }, { - "url": "https://supabase.com/docs/guides/database/secure-data#connecting-your-app-securely", - "title": "Connecting your app securely" + "url": "https://supabase.com/docs/guides/auth/sessions#how-to-ensure-an-access-token-jwt-cannot-be-used-after-a-user-signs-out", + "title": "How to ensure an access token (JWT) cannot be used after a user signs out" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0013_rls_disabled_in_public", - "title": "Database Advisor: Lint 0013_rls_disabled_in_public" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/auth#pricing", + "title": "Pricing" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authuid", - "title": "auth.uid()" + "url": "https://supabase.com/docs/guides/auth#about-authentication-and-authorization", + "title": "About authentication and authorization" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt", - "title": "auth.jwt()" + "url": "https://supabase.com/docs/guides/auth#the-supabase-ecosystem", + "title": "The Supabase ecosystem" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#mfa", - "title": "MFA" + "url": "https://supabase.com/docs/guides/auth#providers", + "title": "Providers" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#bypassing-row-level-security", - "title": "Bypassing Row Level Security" + "url": "https://supabase.com/docs/guides/auth#social-auth", + "title": "Social Auth" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations", - "title": "RLS performance recommendations" + "url": "https://supabase.com/docs/guides/auth#phone-auth", + "title": "Phone Auth" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-indexes", - "title": "Add indexes" + "url": "https://supabase.com/docs/guides/platform/sso", + "title": "Enable SSO for Your Organization" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/platform/sso#how-sso-works-in-supabase", + "title": "How SSO works in Supabase" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#call-functions-with-select", - "title": "Call functions with select" + "url": "https://supabase.com/docs/guides/platform/sso#supported-providers", + "title": "Supported providers" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-1", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/platform/sso#choosing-your-login-flow", + "title": "Choosing your login flow" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-filters-to-every-query", - "title": "Add filters to every query" + "url": "https://supabase.com/docs/guides/platform/sso#idp-initiated-login-recommended", + "title": "IdP-initiated login (recommended)" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-2", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/platform/sso#sp-initiated-login", + "title": "SP-initiated login" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#use-security-definer-functions", - "title": "Use security definer functions" + "url": "https://supabase.com/docs/guides/platform/sso#need-help-choosing", + "title": "Need help choosing?" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#minimize-joins", - "title": "Minimize joins" + "url": "https://supabase.com/docs/guides/platform/sso#key-configuration-options", + "title": "Key configuration options" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-3", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/platform/sso#enabling-sso-for-an-organization", + "title": "Enabling SSO for an organization" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#specify-roles-in-your-policies", - "title": "Specify roles in your policies" + "url": "https://supabase.com/docs/guides/platform/sso#disabling-sso-for-an-organization", + "title": "Disabling SSO for an organization" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-4", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/platform/sso#removing-an-individual-sso-users-access", + "title": "Removing an individual SSO user's access" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#more-resources", - "title": "More resources" + "url": "https://supabase.com/docs/guides/platform/sso#testing-and-best-practices", + "title": "Testing and best practices" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#row-level-security-in-supabase", - "title": "Row Level Security in Supabase" + "url": "https://supabase.com/docs/guides/platform/sso#advanced-scenarios", + "title": "Advanced scenarios" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#policies", - "title": "Policies" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#enabling-row-level-security", - "title": "Enabling Row Level Security" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#openid-connect-oidc", + "title": "OpenID Connect (OIDC)" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#auto-enable-rls-for-new-tables", - "title": "Auto-enable RLS for new tables" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#public-clients-token_endpoint_auth_method-none-1", + "title": "Public clients (token_endpoint_auth_method: none)" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authenticated-and-unauthenticated-roles", - "title": "Authenticated and unauthenticated roles" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#confidential-clients-token_endpoint_auth_method-client_secret_basic-1", + "title": "Confidential clients (token_endpoint_auth_method: client_secret_basic)" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#creating-policies", - "title": "Creating policies" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#confidential-clients-token_endpoint_auth_method-client_secret_post-1", + "title": "Confidential clients (token_endpoint_auth_method: client_secret_post)" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#select-policies", - "title": "SELECT policies" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#example-in-javascript-1", + "title": "Example in JavaScript" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#insert-policies", - "title": "INSERT policies" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#refresh-response", + "title": "Refresh response" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#update-policies", - "title": "UPDATE policies" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#id-tokens", + "title": "ID tokens" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#delete-policies", - "title": "DELETE policies" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#example-id-token", + "title": "Example ID token" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#views", - "title": "Views" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#standard-oidc-claims", + "title": "Standard OIDC claims" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#helper-functions", - "title": "Helper functions" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#userinfo-endpoint", + "title": "UserInfo endpoint" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0017_foreign_table_in_api", - "title": "Database Advisor: Lint 0017_foreign_table_in_api" - } - ], - "resultChars": 203452 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security insert select update delete policies auth.uid bookmarks\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#oidc-discovery", + "title": "OIDC discovery" + }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#token-validation", + "title": "Token validation" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#jwks-endpoint", + "title": "JWKS endpoint" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#validating-tokens", + "title": "Validating tokens" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#what-to-validate", + "title": "What to validate" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#managing-user-grants", + "title": "Managing user grants" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#viewing-authorized-applications", + "title": "Viewing authorized applications" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#revoking-access", + "title": "Revoking access" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#supported-grant-types", + "title": "Supported grant types" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#authorization-code-flow-with-pkce", + "title": "Authorization code flow with PKCE" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#how-it-works", + "title": "How it works" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#flow-diagram", + "title": "Flow diagram" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-1-generate-pkce-parameters", + "title": "Step 1: Generate PKCE parameters" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-2-authorization-request", + "title": "Step 2: Authorization request" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#required-parameters", + "title": "Required parameters" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#optional-parameters", + "title": "Optional parameters" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-3-user-authentication-and-consent", + "title": "Step 3: User authentication and consent" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-4-authorization-code-issued", + "title": "Step 4: Authorization code issued" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-5-token-exchange", + "title": "Step 5: Token exchange" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#public-clients-token_endpoint_auth_method-none", + "title": "Public clients (token_endpoint_auth_method: none)" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#confidential-clients-token_endpoint_auth_method-client_secret_basic", + "title": "Confidential clients (token_endpoint_auth_method: client_secret_basic)" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#confidential-clients-token_endpoint_auth_method-client_secret_post", + "title": "Confidential clients (token_endpoint_auth_method: client_secret_post)" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#example-in-javascript", + "title": "Example in JavaScript" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#step-6-token-response", + "title": "Step 6: Token response" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#access-token-structure", + "title": "Access token structure" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#oauth-specific-claims", + "title": "OAuth-specific claims" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#available-scopes", + "title": "Available scopes" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#refresh-token-flow", + "title": "Refresh token flow" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#when-to-refresh", + "title": "When to refresh" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows#refresh-request", + "title": "Refresh request" } ], - "resultChars": 60041 + "resultChars": 162607 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"auth.uid request.jwt.claims current_setting Supabase Postgres JWT claims\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"auth.users deleted_at Supabase Auth deleted_at banned_until\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", - "title": "RLS Performance and Best Practices" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A", + "title": "Errors when creating / updating / deleting users" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/telemetry/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/auth/rate-limits", + "title": "Rate limits" + }, + { + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" } ], - "resultChars": 103093 + "resultChars": 118546 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"auth.uid implementation request.jwt.claim.sub current_setting\", limit: 3) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"publishable secret API keys Supabase anon service_role RLS\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", + "title": "Build a User Management App with Ionic React" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" } ], - "resultChars": 61357 + "resultChars": 259070 } ] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ + "realtime", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", - "interface": "cli", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": false, - "notes": "avatar_url not found on public.profiles" + "name": "orders table added to supabase_realtime publication", + "passed": true }, { - "name": "migration 20240220000000 is recorded in the remote history", + "name": "courier_locations still in supabase_realtime publication", "passed": true }, { - "name": "remote migration history matches local migration files", + "name": "publication still publishes INSERT events", "passed": true }, { - "name": "local migrations are a valid reconciled sequence", + "name": "RLS still enabled on orders", "passed": true }, { - "name": "production profile data is intact (not reset)", - "passed": true + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Applied via `supabase db push --db-url \"$DB_URL\"` (#22), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding local `supabase/migrations/20240115000000_add_profile_bio.sql` (#20) and then using `supabase db push`/migration workflow; final `supabase migration list --db-url` showed local and remote histories aligned (#23). No disallowed direct SQL mutation or prepared-statement reset workaround seen." + "judgeNotes": "The assistant correctly diagnosed that orders was missing from the supabase_realtime publication, added public.orders via ALTER PUBLICATION, verified courier_locations remained, and did not weaken RLS/policies or blame client/RLS/networking." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/resolve-database-001-migration-history-mismatch.json" + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", "product": [ - "database" + "edge-functions" ], "topic": [ - "observability", - "sql" + "observability" ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, including most/all 8 gateway failures from 07:00Z to 12:00Z." }, { - "name": "created index covering user_id and created_at", - "passed": true + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "The answer does note that 503s appear at the gateway layer with no corresponding function-side traces and unchanged deployment, but it also attributes the issue to a 'runtime/gateway availability problem' and recommends redeploying/rolling back the functions. The rubric says to fail if it recommends fixing or redeploying the function as remediation or blames runtime/function as primary cause." }, { - "name": "query plan uses an index and avoids sequential scan", + "name": "recommended a concrete next step", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true + "judgeNotes": "The assistant recommended concrete actionable next steps, including redeploying/rolling back functions, checking Supabase platform health for the affected timestamps, adding handler/dependency logging, and adding retry/backoff." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike.json" + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", + "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", "product": [ + "data-api", "database", "auth" ], "topic": [ "rls", - "security" + "sdk" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", + "name": "RLS still enabled on bookmarks", "passed": true }, { - "name": "tenant A author can update own note", + "name": "user A reads own bookmarks", "passed": true }, { - "name": "tenant B cannot update org A note", + "name": "user B cannot read user A bookmarks", "passed": true }, { - "name": "tenant B author can delete own note", + "name": "anon reads no bookmarks", "passed": true }, { - "name": "tenant B cannot delete org A note", + "name": "user A can save a new bookmark", "passed": true }, { - "name": "tenant A can insert note in own org", + "name": "user B cannot insert a bookmark as user A", "passed": true }, { - "name": "tenant B cannot insert into org A", - "passed": true + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS enabled with no policies, kept RLS enabled, and added authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant notes workspace\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } ... on ManagementApiReference { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso/auth-sso-saml", - "title": "Single Sign-On with SAML 2.0 for Projects" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/deployment/maturity-model", - "title": "Maturity Model" - } - ], - "resultChars": 97790 - } - ] + "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -10459,50 +10913,42 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ - "database", - "data-api" + "database" ], "topic": [ - "migrations", - "rls" + "migrations" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", + "name": "the avatar_url column is applied on the hosted profiles table", "passed": true }, { - "name": "todos table is created by a migration file", + "name": "migration 20240220000000 is recorded in the remote history", "passed": true }, { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", + "name": "remote migration history matches local migration files", "passed": true }, { - "name": "a SELECT policy targets the authenticated role", + "name": "local migrations are a valid reconciled sequence", "passed": true }, { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" + "name": "production profile data is intact (not reset)", + "passed": true }, { - "name": "REST API returns the todos to authenticated requests", + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "notes": "2 rows" + "judgeNotes": "Applied pending avatar_url via `supabase db push` (#20), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled orphan bio migration by adding local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` (#17), after which `supabase migration list` showed local and remote aligned (#18/#22). No disallowed workaround seen; psql usage was read-only inspection." } ], "skills": { @@ -10512,10 +10958,10 @@ "docs": { "calls": [] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-001-bootstrap-app.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -10526,33 +10972,38 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ "database" ], "topic": [ - "declarative-schema", - "migrations" + "observability", + "sql" ], "suite": "benchmark", - "interface": "cli", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": false + "name": "inspected pg_stat_statements for query performance", + "passed": true }, { - "name": "schema file updated to include description column", + "name": "ran EXPLAIN on the expensive query", "passed": true }, { - "name": "a new migration was generated for the change", + "name": "created index covering user_id and created_at", "passed": true }, { - "name": "description column exists in the live database", + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", "passed": true } ], @@ -10563,10 +11014,10 @@ "docs": { "calls": [] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-002-declarative-schema.json" + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -10577,36 +11028,55 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", "product": [ "database", - "edge-functions", - "cron", - "queues" + "auth" ], "topic": [ - "sql", - "sdk" + "rls", + "security" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "name": "RLS enabled on notes", + "passed": true }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "name": "tenant A sees only org A notes", + "passed": true }, { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true } ], "skills": { @@ -10614,2392 +11084,346 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pgmq create queue send pop delete archive read Supabase cron schedule\", limit: 5) {\n totalCount\n nodes {\n __typename\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - } - ], - "resultChars": 28867 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron schedule cron.schedule cron.unschedule Supabase SQL job name\", limit: 5) {\n totalCount\n nodes {\n __typename\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - } - ], - "resultChars": 33057 - } - ] + "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "database" + "database", + "data-api" ], "topic": [ - "migrations" + "migrations", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", + "name": "supabase project initialised (supabase/config.toml exists)", "passed": true }, { - "name": "row counts match (teams=5, members=10, tasks=13)", + "name": "todos table is created by a migration file", "passed": true }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" }, { - "name": "tasks_team_status_idx index survived the restore", + "name": "row level security is enabled on todos", "passed": true }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "a SELECT policy targets the authenticated role", "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" }, { - "name": "user A cannot force-read user B note", + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "status=200" + "notes": "0 rows" }, { - "name": "user B cannot force-read user A note", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variable SUPABASE_ANON_KEY auth user in Deno\", limit: 5) { nodes { title href content ... on Guide { subsections { totalCount } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - } - ], - "resultChars": 42100 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing API_URL/SECRET_KEY/PUBLISHABLE_KEY — new API keys are required for @supabase/server; is the stack running on a new-enough CLI? got keys: ANON_KEY, DB_URL, JWT_SECRET, PUBLISHABLE_KEY, SECRET_KEY, SERVICE_ROLE_KEY" + "notes": "2 rows" } ], "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions verifyJWT apikey service role auth user access token Deno.env.get SUPABASE_SERVICE_ROLE_KEY\", limit: 5) { edges { node { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href language methodName content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 55962 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions failed to determine entrypoint index.ts config.toml entrypoint\", limit: 5) { edges { node { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 58296 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"failed to determine entrypoint supabase functions serve local function layout index.ts\", limit: 10) { edges { node { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/unable-to-deploy-edge-function", - "title": "Unable to deploy Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", - "title": "Issues serving Edge Functions locally" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" - } - ], - "resultChars": 60734 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019f6c9d-552b-73d4-b26e-66eca00b3f0c/receipt-alpha.pdf, 019f6c9d-552b-73d4-b26e-66eca00b3f0c/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets rubric: private user-files bucket, owner-scoped authenticated SELECT and INSERT policies with WITH CHECK, RLS kept enabled, and supabase-js createSignedUrl with short expiry." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage.objects policy foldername signed url supabase-js createSignedUrl\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads#signing-urls", - "title": "Signing URLs" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads#public-buckets", - "title": "Public buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads#downloading", - "title": "Downloading" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads#programmatic-downloads-with-query-parameters", - "title": "Programmatic downloads with query parameters" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads#private-buckets", - "title": "Private buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads#overwriting-files", - "title": "Overwriting files" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads#presigned-uploads", - "title": "Presigned uploads" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads#uppy-example", - "title": "Uppy example" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads#concurrency", - "title": "Concurrency" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads#upload-url", - "title": "Upload URL" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#storage-api-configuration", - "title": "Storage API configuration:" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#imgproxy-configuration", - "title": "imgproxy configuration:" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#self-hosting", - "title": "Self hosting" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#supported-image-formats", - "title": "Supported image formats" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#limits", - "title": "Limits" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#modes", - "title": "Modes" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#resizing", - "title": "Resizing" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#optimizing", - "title": "Optimizing" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#transformation-options", - "title": "Transformation options" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#nextjs-loader", - "title": "Next.js loader" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#automatic-image-optimization-webp", - "title": "Automatic image optimization (WebP)" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#downloading-images", - "title": "Downloading images" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#signing-urls-with-transformation-options", - "title": "Signing URLs with transformation options" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#get-a-public-url-for-a-transformed-image", - "title": "Get a public URL for a transformed image" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations#manage-image-transformations", - "title": "Manage image transformations" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#exceeding-quotas", - "title": "Exceeding Quotas" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#optimize-usage", - "title": "Optimize usage" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#view-usage", - "title": "View usage" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#billing-examples", - "title": "Billing examples" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#exceeding-quota", - "title": "Exceeding quota" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#usage-on-your-invoice", - "title": "Usage on your invoice" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#what-you-are-charged-for", - "title": "What you are charged for" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#example", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#how-charges-are-calculated", - "title": "How charges are calculated" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#example-1", - "title": "Example" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations#within-quota", - "title": "Within quota" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - } - ], - "resultChars": 103502 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"storage.foldername storage.objects policy path tokens auth.uid\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 50894 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "3 passed, 1 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a tenant isolation flaw, specifically that members can read cross-org posts, and grounds the conclusion in the pgTAP result where the cross-org `posts` check returned 1 row instead of 0. It also distinguishes that `notes` passed." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": false, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Fails: Supabase scrape uses basic_auth.password instead of password_file, docker-compose.yml does not mount the password file via volume or Compose secret, and README instructs replacing the value with a Secret API key in prometheus.yml. Existing app scrape and HTTPS metrics path are present." - }, - { - "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README gives basic Supabase scrape setup and restart guidance, but fails required criteria: it instructs replacing the password inline rather than placing a matching secret file, does not configure/use a secret file, and lacks concrete verification steps via Prometheus targets, PromQL/Grafana, or equivalent." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"metrics Prometheus project metrics\", limit: 10) { edges { node { __typename title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", - "title": "Manual replication monitoring" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - } - ], - "resultChars": 35779 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Edge Functions secrets set env-file deploy cli project-ref\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/push-notifications", - "title": "Sending Push Notifications" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" - } - ], - "resultChars": 40692 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"functions deploy no-verify-jwt config.toml verify_jwt false\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/upstash-redis", - "title": "Upstash Redis" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - } - ], - "resultChars": 22399 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", - "product": [ - "auth" - ], - "topic": [ - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "victim session active before delete-account", - "passed": true - }, - { - "name": "delete_account flow ran for the victim", - "passed": true - }, - { - "name": "delete-account revokes the user's sessions", - "passed": true, - "notes": "sessions left: 0" - }, - { - "name": "deleted user's refresh token is rejected", - "passed": true - }, - { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" - }, - { - "name": "other users keep their sessions and access", - "passed": true - }, - { - "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "Fails because it does not explain that existing access tokens are stateless JWTs that remain cryptographically valid until expiry for purely local validation such as getClaims() or custom JWT middleware. It instead says the only remaining race is an in-flight request, which omits the required caveat. The diagnosis, RLS/data-path fix, session/refresh-token revocation, and publishable vs secret key explanation are otherwise largely correct." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"publishable secret keys RLS anon service_role\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on ClientLibraryFunctionReference { title href methodName language content }\n ... on ManagementApiReference { title href content }\n ... on CLICommandReference { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - } - ], - "resultChars": 112269 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Identified orders missing from supabase_realtime publication as root cause, added public.orders with ALTER PUBLICATION via migration, and preserved courier_locations/RLS/policies." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant identified `image-transform` as the affected function and described a recurring morning pattern of gateway-level HTTP 503s on 2026-04-28, including alternating/intermittent failures across roughly 06:00–12:00 UTC. This satisfies the rubric despite mentioning an additional avatar-upload issue." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "The assistant notes that the 503s lack function-side execution details and likely failed before runtime, but it also frames this as a function-level/edge-function issue and recommends redeploying the functions. The rubric explicitly fails answers that recommend fixing or redeploying the function as remediation." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps: redeploying functions, adding retry/backoff, instrumenting packages, correlating a specific 500 with request payload/time, and escalating to Supabase support if 503s continue." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push --db-url \"$(cat supabase/.temp/pooler-url)\" --include-all` (#35), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` (#34) and then letting the successful `supabase db push` align local/remote history; `supabase migration list` (#36) showed all three migrations matched. Read-only psql inspections were used; no forbidden direct SQL mutation or prepared-statement workaround observed." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security authenticated role SELECT policy anon authenticated Data API grants local development migrations seed.sql\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - } - ], - "resultChars": 104061 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase db diff local generate migration imperative migrations\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - } - ], - "resultChars": 45083 - } - ] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"declarative database schemas local development schema_paths db diff migration\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-diff", - "title": "Diffs the local database for schema changes" - } - ], - "resultChars": 34329 - } - ] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete message Edge Function Cron pg_cron local\", limit: 8) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", - "title": "Invoke an Edge Function every minute" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", - "title": "How does Cron work?" - }, - { - "url": "https://supabase.com/docs/guides/cron#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", - "title": "pgmq_public.send(queue_name, message, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", - "title": "pgmq_public.pop(queue_name)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", - "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", - "title": "pgmq_public.archive(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", - "title": "pgmq_public.delete(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", - "title": "pgmq_public.read(queue_name, sleep_seconds, n)" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/cron/install#uninstall", - "title": "Uninstall" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", - "title": "http_post" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", - "title": "http_get" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", - "title": "http_delete" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", - "title": "Signature " - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", - "title": "Inspecting request data" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", - "title": "Inspecting failed requests" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", - "title": "Get current settings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", - "title": "Alter settings" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", - "title": "Examples" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", - "title": "Invoke a Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", - "title": "Call an endpoint every minute with pg_cron" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", - "title": "Execute pg_net in a trigger" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", - "title": "Send multiple table rows in one request" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", - "title": "Limitations" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", - "title": "Debugging requests" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", - "title": "Analyzing responses" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", - "title": "Usage " - } - ], - "resultChars": 116291 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Queues API pgmq_public.read delete send create queue SQL\", limit: 10) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", - "title": "pgmq_public.read(queue_name, sleep_seconds, n)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", - "title": "pgmq_public.delete(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", - "title": "pgmq_public.archive(queue_name, message_id)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", - "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", - "title": "pgmq_public.send(queue_name, message, sleep_seconds)" - }, - { - "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", - "title": "pgmq_public.pop(queue_name)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#drop_queue", - "title": "drop_queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#deletingarchiving-messages", - "title": "Deleting/Archiving messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#delete-single", - "title": "delete (single)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#delete-batch", - "title": "delete (batch)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#purge_queue", - "title": "purge_queue" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#archive-single", - "title": "archive (single)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#archive-batch", - "title": "archive (batch)" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#utilities", - "title": "Utilities" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#set_vt", - "title": "set_vt" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#list_queues", - "title": "list_queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#metrics", - "title": "metrics" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#metrics_all", - "title": "metrics_all" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#types", - "title": "Types" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#message_record", - "title": "message_record" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#get-usage", - "title": "Usage " - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#queue-management", - "title": "Queue management" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#create", - "title": "create" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#create_unlogged", - "title": "create_unlogged" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#detach_archive", - "title": "detach_archive" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#sending-messages", - "title": "Sending messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#send", - "title": "send" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#send_batch", - "title": "send_batch" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#reading-messages", - "title": "Reading messages" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#read", - "title": "read" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#read_with_poll", - "title": "read_with_poll" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq#pop", - "title": "pop" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues#stop-exposing-queues", - "title": "Stop exposing queues" - }, + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security policy authenticated users select table anon no access grant Data API migrations local development\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues#expose-queues-with-docker-compose", - "title": "Expose queues with Docker compose" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues#expose-queues-with-supabase-cli", - "title": "Expose Queues with Supabase CLI" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 278303 - }, + "resultChars": 135961 + } + ] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"site:supabase.com/docs/guides/queues/api delete message_id read sleep_seconds send messages\", limit: 6) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"local development declarative database schemas generate migration db diff schema_paths\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", - "title": "Consuming messages in an Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", - "title": "Enable RLS on your tables in pgmq schema" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", - "title": "Pull-Based Queue" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#message", - "title": "Message" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", - "title": "Queue types" + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", - "title": "Create Queues" + "url": "https://supabase.com/docs/reference/cli/supabase-db-diff", + "title": "Diffs the local database for schema changes" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", - "title": "What happens when you create a queue?" - }, + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + } + ], + "resultChars": 72112 + } + ] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 1 -> 2" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 6) from the queue" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function JavaScript cron schedule every minute\", limit: 8) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", - "title": "Expose Queues to client-side consumers" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", - "title": "Grant permissions to pgmq_public database functions" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", - "title": "Enqueueing and dequeueing messages" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { "url": "https://supabase.com/docs/guides/queues", "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/queues#features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/queues#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#framework-examples", - "title": "Framework examples" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#2-initialize-the-client", - "title": "2. Initialize the client" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#1-install-the-client-library", - "title": "1. Install the client library" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#quick-start", - "title": "Quick start" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#get-api-details", - "title": "Get API details" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#3-create-your-first-channel", - "title": "3. Create your first Channel" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#4-set-up-authorization", - "title": "4. Set up authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#5-send-and-receive-messages", - "title": "5. Send and receive messages" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#51-using-client-libraries", - "title": "5.1 using client libraries" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#52-using-httprest-api", - "title": "5.2 using HTTP/REST API" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#53-using-database-triggers", - "title": "5.3 using database triggers" - }, - { - "url": "https://supabase.com/docs/guides/realtime/getting_started#use-private-channels", - "title": "Use private channels" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#essential-best-practices", - "title": "Essential best practices" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#follow-naming-conventions", - "title": "Follow naming conventions" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#clean-up-subscriptions", - "title": "Clean up subscriptions" - }, + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + } + ], + "resultChars": 45196 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Queues quickstart create queue pgmq.create SQL pop removes message pgmq_public service_role\", limit: 6) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/getting_started#choose-the-right-feature", - "title": "Choose the right feature" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-broadcast", - "title": "When to use Broadcast" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-presence", - "title": "When to use Presence" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-postgres-changes", - "title": "When to use Postgres Changes" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#next-steps", - "title": "Next steps" - }, + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" + } + ], + "resultChars": 39503 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY default secrets createClient Deno env\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/getting_started#core-features", - "title": "Core features" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#security--configuration", - "title": "Security & configuration" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#advanced-topics", - "title": "Advanced topics" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#integration-guides", - "title": "Integration guides" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", - "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" } ], - "resultChars": 110913 + "resultChars": 48093 } ] }, @@ -13063,31 +11487,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restore existing Postgres pg_dump custom format local Supabase CLI\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"restore existing Postgres database pg_restore local Supabase binary custom dump roles ownership\", limit: 5) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Debugging and monitoring" + "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", + "title": "Transferring from platform to self-hosted Supabase" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/remove-superuser-access", + "title": "Remove superuser access from Studio" } ], - "resultChars": 52540 + "resultChars": 38293 } ] }, @@ -13160,46 +11584,35 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions JWT verification authenticate user getClaims Authorization header createClient RLS\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient getUser RLS service role\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, { "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 79335 + "resultChars": 67563 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions verify_jwt asymmetric signing keys recommended authentication getClaims 2026\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Functions authorization header getUser SUPABASE_ANON_KEY createClient user JWT\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -13207,31 +11620,27 @@ "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { "url": "https://supabase.com/docs/guides/functions/auth-headers", "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" } ], - "resultChars": 96675 + "resultChars": 55755 } ] }, @@ -13274,27 +11683,27 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"9d427975-63c5-47ed-b72b-5b56288341f8\",\"metric\":\"steps_a_mrpdsmvt\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"80d58873-2aec-45aa-8a73-40c65ba02b42\",\"metric\":\"steps_a_mry6awoq\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"9d427975-63c5-47ed-b72b-5b56288341f8\",\"metric\":\"steps_a_mrpdsmvt\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"80d58873-2aec-45aa-8a73-40c65ba02b42\",\"metric\":\"steps_a_mry6awoq\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"a28cbd5f-63ce-4a79-aabe-16254a9f2708\",\"metric\":\"steps_b_mrpdsmvt\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"25a25df5-daa2-4f8b-b396-75627f0b141c\",\"metric\":\"steps_b_mry6awoq\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "rejects an unverified (forged) user token", @@ -13304,7 +11713,7 @@ { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { "name": "implementation uses @supabase/server", @@ -13325,47 +11734,95 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authentication JWT verification apikey secret key service role getUser createClient Deno environment SUPABASE_SERVICE_ROLE_KEY\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions JWT verification service role apikey header createClient Authorization user getUser verify_jwt config\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 59788 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Securing Edge Functions API key secret key authenticate apikey service role compare Deno env\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/security/npm-security", + "title": "Securing npm installs" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" + }, + { + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" } ], - "resultChars": 76653 + "resultChars": 133701 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server verifyCredentials apikey auth user secret key Edge Functions createContextClient createAdminClient\", limit: 10) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Securing Edge Functions dual authentication user JWT secret API key SUPABASE_SECRET_KEYS apikey handler\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -13373,82 +11830,131 @@ "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/migrating-to-ssr-from-auth-helpers", - "title": "Migrating to the SSR package from Auth Helpers" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" } ], - "resultChars": 71941 + "resultChars": 170820 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth secret legacy service_role JWT apikey @supabase/server INVALID_CREDENTIALS\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase config.toml functions entrypoint index.ts failed to determine entrypoint\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, + { + "url": "https://supabase.com/docs/guides/functions/dependencies", + "title": "Managing dependencies" + } + ], + "resultChars": 49231 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"@supabase/server withSupabase legacy service_role key auth mode apikey\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-svelte", + "title": "Build a User Management App with Svelte" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" } ], - "resultChars": 79252 + "resultChars": 169829 } ] }, @@ -13495,7 +12001,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019f71b9-0677-7412-9c1e-57a03e47da0e/receipt-alpha.pdf, 019f71b9-0677-7412-9c1e-57a03e47da0e/receipt-beta.pdf" + "notes": "saw: 019f9166-bfee-75dd-875e-cabb85d637c9/receipt-alpha.pdf, 019f9166-bfee-75dd-875e-cabb85d637c9/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -13516,7 +12022,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Creates private user-files bucket, keeps RLS policies on storage.objects scoped to authenticated users and auth.uid() folder ownership for SELECT and INSERT (plus update/delete), and provides supabase-js createSignedUrl with expiry. No public bucket/getPublicUrl/service-role client-side issue." + "judgeNotes": "Meets all requirements: private user-files bucket, RLS remains enabled, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, and supabase-js createSignedUrl with a 15-minute expiry." } ], "skills": { @@ -13532,95 +12038,94 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage access control policy foldername auth.uid private bucket createSignedUrl JavaScript\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Storage access control RLS policies foldername auth uid createSignedUrl private bucket\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/storage", - "title": "Storage" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/firebase-storage", - "title": "Migrated from Firebase Storage to Supabase" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" } ], - "resultChars": 18276 + "resultChars": 45736 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policy storage.foldername(name) auth.uid upload select update delete\", limit: 6) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"storage.foldername(name) auth.uid storage.objects policy first folder user id\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, { "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/storage/quickstart", + "title": "Storage Quickstart" }, { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + }, + { + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" } ], - "resultChars": 25062 + "resultChars": 17732 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createSignedUrl storage expiresIn private bucket supabase-js\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"JavaScript createSignedUrl expiresIn storage private bucket create signed URL\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, { "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", "title": "Storage Image Transformations" }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" } ], - "resultChars": 32239 + "resultChars": 26114 } ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access.json" }, { @@ -13648,17 +12153,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "8 passed, 0 failed" + "notes": "13 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identified `posts` as having a tenant isolation flaw, specifically that organization members could read posts across tenants, and grounded this in pgTAP failures. It also correctly stated `notes` isolation was enforced. Extra mention of `memberships` does not undermine the required conclusion." + "judgeNotes": "The agent explicitly identified `posts` as having the tenant isolation flaw: members could read every organization because the policy omitted the `org_id` comparison. It grounded this in pgTAP results, noting baseline failures and post-fix pass results. Although it also mentioned other issues, it did not blame `notes` instead of `posts` or dismiss test results." } ], "skills": { @@ -13674,7 +12179,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security testing auth.uid tenant isolation policies pgTAP\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security testing pgTAP auth.uid tenant isolation policies\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -13686,19 +12191,19 @@ "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" } ], - "resultChars": 56902 + "resultChars": 81583 } ] }, @@ -13773,46 +12278,6 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search gte-small vector dimensions match_documents RPC row level security auth uid function security invoker\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/introduction", - "title": "Vector Buckets" - }, - { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" - }, - { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 111855 - }, { "source": "web_search", "query": "https://supabase.com/changelog.md", @@ -13822,33 +12287,38 @@ } ] }, + { + "source": "web_search", + "query": "site:supabase.com/changelog pgvector breaking change Supabase 2025 2026", + "pages": [] + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase vector indexes HNSW vector_ip_ops normalized embeddings inner product 384 gte-small\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"gte-small pgvector 384 dimensions semantic search match function RLS security invoker\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" } ], - "resultChars": 58512 + "resultChars": 78435 } ] }, @@ -13884,12 +12354,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets all requirements: preserves app scrape, adds HTTPS Supabase Metrics API scrape at /customer/v1/privileged/metrics for .supabase.co, uses basic_auth with password_file, and wires the password file via a Compose secret." + "judgeNotes": "Prometheus preserves the app scrape and adds an HTTPS Supabase scrape using /customer/v1/privileged/metrics with HTTP Basic Auth and password_file. docker-compose mounts the same password file path read-only. No bearer auth or hardcoded secret key is present." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes required live setup steps: project ref, creating/copying a Supabase Secret API key, placing it in the Compose secret file, restarting the Compose stack, and concrete verification via Prometheus targets and PromQL/Grafana." + "judgeNotes": "README includes concrete live setup: replace project ref, create/copy Supabase Secret API key, write it to the mounted secret file path matching docker-compose/prometheus.yml, and recreate/reload Prometheus. Verification includes direct curl, Prometheus targets, API target check, and Grafana Explore." } ], "skills": { @@ -13905,159 +12375,39 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase project metrics Prometheus endpoint authentication service role metrics\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on ManagementApiReference { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"project metrics Prometheus Grafana metrics endpoint service role authentication\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics#choose-your-monitoring-stack", - "title": "Choose your monitoring stack" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics#what-you-can-do-with-the-metrics-api", - "title": "What you can do with the Metrics API" - }, { "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#4-configure-alerting", - "title": "4. Configure alerting" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#3-import-supabase-dashboards", - "title": "3. Import Supabase dashboards" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#2-deploy-grafana", - "title": "2. Deploy Grafana" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#5-operating-tips", - "title": "5. Operating tips" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#1-deploy-prometheus", - "title": "1. Deploy Prometheus" - }, { "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#5-troubleshooting", - "title": "5. Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#prerequisites", - "title": "Prerequisites" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#1-create-a-grafana-cloud-stack", - "title": "1. Create a Grafana Cloud stack" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#2-configure-the-supabase-integration", - "title": "2. Configure the Supabase integration" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#3-import-the-supabase-dashboard", - "title": "3. Import the Supabase dashboard" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#4-configure-alerts-optional", - "title": "4. Configure alerts (optional)" - }, { "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#5-multi-project-setups", - "title": "5. Multi-project setups" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#3-downstream-dashboards", - "title": "3. Downstream dashboards" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#2-secure-the-credentials", - "title": "2. Secure the credentials" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#collector-specific-notes", - "title": "Collector-specific notes" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#1-define-the-scrape-job", - "title": "1. Define the scrape job" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#components", - "title": "Components" - }, - { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#4-alerts-and-automation", - "title": "4. Alerts and automation" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas#pricing", - "title": "Pricing" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas#centralized-configuration-management", - "title": "Centralized configuration management" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas#metrics", - "title": "Metrics" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas#logging", - "title": "Logging" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas#querying-through-the-sql-editor", - "title": "Querying through the SQL editor" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas#api-load-balancer", - "title": "API load balancer" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas#dedicated-connection-pool", - "title": "Dedicated connection pool" + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas#dedicated-endpoints", - "title": "Dedicated endpoints" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas#features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", + "title": "PGAudit: Postgres Auditing" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas#about-read-replicas", - "title": "About Read Replicas" + "url": "https://supabase.com/docs/guides/troubleshooting/grafana-not-displaying-data-sXJrMj", + "title": "Grafana not displaying data" } ], - "resultChars": 96198 + "resultChars": 67164 } ] }, @@ -14099,7 +12449,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -14119,31 +12469,35 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions deploy secrets environment variables Deno.env WEATHER_API_KEY CORS invoke browser\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy verify_jwt CORS Deno serve WEATHER_API_KEY\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", "title": "Inspecting edge function environment variables" }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" }, - { - "url": "https://supabase.com/docs/guides/functions/cors", - "title": "CORS (Cross-Origin Resource Sharing) support for Invoking from the browser" - }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" } ], - "resultChars": 37347 + "resultChars": 41860 } ] }, @@ -14205,7 +12559,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose install production secrets JWT keys API keys SMTP S3 docker compose\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"self-hosting Docker compose production secrets JWT keys dashboard basic auth SMTP S3 analytics vector\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -14216,56 +12570,32 @@ "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", "title": "Envoy API Gateway" }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", - "title": "Configure Phone Login & MFA" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" - } - ], - "resultChars": 174690 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosted proxy HTTPS Caddy PROXY_DOMAIN ports firewall Kong database Supavisor production Docker\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/enable-mcp", - "title": "Enabling MCP Server Access" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" } ], - "resultChars": 77431 + "resultChars": 178705 } ] }, @@ -14294,7 +12624,7 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "victim session active before delete-account", @@ -14315,8 +12645,7 @@ }, { "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "passed": true }, { "name": "other users keep their sessions and access", @@ -14325,7 +12654,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements real revocation by banning the auth user and deleting sessions, and also adds RLS active-user checks that consistently justify no post-deletion Data API window for protected tables while caveating JWTs remain valid for local validation/unprotected paths until exp. It correctly distinguishes publishable vs secret keys and says secret bypasses RLS and is server-only." + "judgeNotes": "Meets rubric: identifies soft-delete-only root cause, implements real auth/session removal plus RLS active-session mitigation, explains JWT validity/window consistently with the implemented RLS closure, and correctly distinguishes publishable frontend key vs secret server-only RLS-bypassing key." } ], "skills": { @@ -14341,191 +12670,79 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user existing access token JWT session revoke auth.sessions delete_account publishable secret keys RLS\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"delete user does not invalidate access token sessions revoke refresh tokens sign out admin deleteUser\", limit: 6) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/storage/s3/authentication", - "title": "S3 Authentication" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" }, { - "url": "https://supabase.com/docs/guides/storage/management/delete-objects", - "title": "Delete Objects" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" } ], - "resultChars": 69441 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog Supabase changelog auth session delete user API keys publishable secret", - "pages": [] + "resultChars": 40176 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase publishable key secret key legacy anon service_role RLS bypass behavior\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"publishable key secret key legacy anon service_role RLS bypass new API keys\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/storage/security/access-control", "title": "Storage Access Control" }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - } - ], - "resultChars": 147735 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Understanding API keys publishable key secret key sb_publishable sb_secret service_role bypass RLS\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - } - ], - "resultChars": 64041 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"sign out revoke sessions access token remains valid until expiry Supabase Auth delete user\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso", - "title": "Enable SSO for Your Organization" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/platform/temporary-access", - "title": "Temporary access" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" } ], - "resultChars": 143995 + "resultChars": 134247 } ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "codex-gpt-5.6/investigate-auth-001-deleted-user-access.json" }, { @@ -14574,7 +12791,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that the subscribed channel was silent because public.orders was missing from the supabase_realtime publication while courier_locations was present, added only public.orders to the existing publication via ALTER PUBLICATION inside an idempotent migration, and preserved RLS/policies and courier_locations. It did not blame RLS/client/networking or weaken security." + "judgeNotes": "The assistant correctly identified orders missing from supabase_realtime as the cause, added only public.orders via ALTER PUBLICATION, verified courier_locations remained published, and did not alter RLS/policies or blame other causes." } ], "skills": { @@ -14590,7 +12807,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes Realtime publication supabase_realtime add table postgres_changes subscribe INSERT\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Postgres Changes enable table supabase_realtime publication ALTER PUBLICATION\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -14602,19 +12819,19 @@ "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" + "url": "https://supabase.com/docs/guides/database/replication/pipelines", + "title": "Set up Pipelines" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" } ], - "resultChars": 126779 + "resultChars": 121815 }, { "source": "web_search", @@ -14627,7 +12844,7 @@ }, { "source": "web_search", - "query": "site:supabase.com/changelog realtime postgres changes publication breaking change Supabase", + "query": "site:supabase.com/changelog Realtime Postgres Changes publication breaking change", "pages": [] } ] @@ -14661,17 +12878,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as affected and described eight intermittent 503s across the morning of 2026-04-28, matching the required recurring gateway failure pattern." + "judgeNotes": "Identified image-transform as affected and described the recurring eight HTTP 503s across 07:00–12:00 UTC on 2026-04-28." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the image-transform 503s to the gateway/platform layer, grounded in gateway-only logs with no Edge Function executions and unchanged deployment with nearby successful invocations. It also advises not to redeploy/change function code." + "judgeNotes": "Attributes 503s to gateway/pre-handler platform path and grounds it in gateway-only logs with no Edge Function execution logs, nearby successes on same deployment, and distinction from avatar-upload's function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete actionable next steps, including opening a Supabase Support case with timestamps and evidence of gateway 503s without execution IDs, preserving request IDs/timestamps/region, adding retries, and avoiding unnecessary redeploys." + "judgeNotes": "The assistant recommended concrete next steps, including querying failed gateway request IDs, opening a Supabase support case with IDs and timestamps, testing cold starts, adding retries, and investigating the isolated 500." } ], "skills": { @@ -14687,7 +12904,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions 503 gateway request never reaches function runtime causes logs\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions 503 gateway request not reaching function logs troubleshooting retries status page\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { @@ -14707,21 +12924,11 @@ "title": "Edge Function 404 error response" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-500-error-response", + "title": "Edge Function 500 error response" } ], - "resultChars": 36301 - }, - { - "source": "web_search", - "query": "site:status.supabase.com April 28 2026 Edge Functions incident 503", - "pages": [] - }, - { - "source": "web_search", - "query": "site:status.supabase.com/incidents \"Apr 28, 2026\" Supabase", - "pages": [] + "resultChars": 40011 } ] }, @@ -14781,7 +12988,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as the cause of zero Data API rows, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK respectively. It also verified owner-only access." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all/zero rows, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts. No permissive or anon/public policies were used." } ], "skills": { @@ -14797,31 +13004,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security auth.uid select insert policy Data API grants authenticated\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security auth.uid select insert policy authenticated Data API grants bookmarks\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, { "url": "https://supabase.com/docs/guides/getting-started/features", "title": "Features" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" } ], - "resultChars": 100048 + "resultChars": 74686 }, { "source": "web_search", @@ -14834,7 +13041,7 @@ }, { "source": "web_search", - "query": "site:supabase.com/changelog breaking change RLS Data API Supabase", + "query": "site:supabase.com/changelog breaking change RLS Data API policies grants", "pages": [] } ] @@ -14888,7 +13095,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Applied pending avatar_url migration with `supabase db push --db-url \"$DB_URL\"` in #9. Reconciled orphan bio migration by adding local `supabase/migrations/20240115000000_add_profile_bio.sql`, then using Supabase CLI push workflow; final migration list shows local/remote histories aligned. No prohibited direct SQL mutation or prepared-statement reset workaround seen." + "judgeNotes": "Applied pending avatar_url migration via Supabase CLI command `supabase db push --db-url \"$DB_URL\" --yes` in #11, with verification showing remote history includes `20240220000000`. Reconciled orphan bio migration by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #9 and then running the same `supabase db push` so local/remote history aligned. No forbidden workaround or direct SQL mutation of remote history/avatar column observed." } ], "skills": { @@ -14896,21 +13103,15 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI database migrations db push repair migration history linked project\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI db push migration history repair linked project deploy migrations\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - }, { "url": "https://supabase.com/docs/guides/deployment/database-migrations", "title": "Database Migrations" @@ -14923,6 +13124,10 @@ "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", "title": "Repair the migration history table" }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" + }, { "url": "https://supabase.com/docs/guides/local-development/database-migrations", "title": "Database migrations" @@ -14934,7 +13139,7 @@ }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6/resolve-database-001-migration-history-mismatch.json" }, { @@ -14974,7 +13179,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -14994,31 +13199,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres query performance pg_stat_statements EXPLAIN ANALYZE composite index order by limit\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"database query optimization composite indexes ORDER BY LIMIT recent rows\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Debugging and monitoring" + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" }, { - "url": "https://supabase.com/docs/guides/database/extensions/hypopg", - "title": "HypoPG: Hypothetical indexes" + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/storage/analytics/query-with-postgres", + "title": "Query with Postgres" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/running-explain-analyze-on-functions", - "title": "Running EXPLAIN ANALYZE on functions" + "url": "https://supabase.com/docs/guides/telemetry/advanced-log-filtering", + "title": "Advanced Log Querying and Filtering" } ], - "resultChars": 34904 + "resultChars": 32369 } ] }, @@ -15100,7 +13305,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant policy auth.uid membership workspace notes\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"row level security multi tenant workspace policy auth uid security definer\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -15108,23 +13313,37 @@ "title": "Features" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/database/tables", + "title": "Tables and Data" } ], - "resultChars": 61519 + "resultChars": 101839 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking change RLS policies", + "pages": [] } ] }, @@ -15196,31 +13415,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security authenticated role SELECT policy anon no rows migrations seed data local development\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"Postgres Row Level Security authenticated users select policy anon no rows Supabase migrations seed data\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, { "url": "https://supabase.com/docs/guides/getting-started/features", "title": "Features" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" } ], - "resultChars": 100768 + "resultChars": 71000 } ] }, @@ -15276,13 +13495,33 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI create database migration alter table add column local stack migration up\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI create migration alter table add column local database migration\", limit: 3) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, { "url": "https://supabase.com/docs/guides/local-development/database-migrations", "title": "Database migrations" }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + } + ], + "resultChars": 54403 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"declarative database schemas schema_paths supabase db diff migration local\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" + }, { "url": "https://supabase.com/docs/guides/deployment/database-migrations", "title": "Database Migrations" @@ -15292,7 +13531,7 @@ "title": "Local development workflow" } ], - "resultChars": 54403 + "resultChars": 55139 } ] }, @@ -15334,12 +13573,12 @@ { "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "queue depth 1 -> 2" + "notes": "queue depth 0 -> 1" }, { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "notes": "function removed the seeded message (id 4) from the queue" } ], "skills": { @@ -15350,978 +13589,894 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send message cron schedule Edge Function read delete messages\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete message pg_cron schedule every minute Edge Function local\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { title href content language methodName } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions#invoke-an-edge-function-every-minute", + "title": "Invoke an Edge Function every minute" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/queues#features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_get", + "title": "http_get" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_post", + "title": "http_post" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature", + "title": "Signature " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#debugging-requests", + "title": "Debugging requests" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#analyzing-responses", + "title": "Analyzing responses" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#delete-usage", + "title": "Usage " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-usage", + "title": "Usage " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#http_delete", + "title": "http_delete" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#post-signature-1", + "title": "Signature " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-request-data", + "title": "Inspecting request data" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#inspecting-failed-requests", + "title": "Inspecting failed requests" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#configuration", + "title": "Configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-current-settings", + "title": "Get current settings" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#alter-settings", + "title": "Alter settings" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#limitations", + "title": "Limitations" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#invoke-a-supabase-edge-function", + "title": "Invoke a Supabase Edge Function" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#call-an-endpoint-every-minute-with-pg_cron", + "title": "Call an endpoint every minute with pg_cron" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#execute-pg_net-in-a-trigger", + "title": "Execute pg_net in a trigger" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#send-multiple-table-rows-in-one-request", + "title": "Send multiple table rows in one request" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-usage", + "title": "Usage " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net#get-signature", + "title": "Signature " + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/cron#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/cron#how-does-cron-work", + "title": "How does Cron work?" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#edit-a-job", + "title": "Edit a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-stored-procedure", + "title": "Call a database stored procedure" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#call-a-database-function-every-5-minutes", + "title": "Call a database function every 5 minutes" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#run-a-vacuum-every-day", + "title": "Run a vacuum every day" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#delete-data-every-week", + "title": "Delete data every week" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#inspecting-job-runs", + "title": "Inspecting job runs" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#unschedule-a-job", + "title": "Unschedule a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#activatedeactivate-a-job", + "title": "Activate/Deactivate a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#schedule-a-job", + "title": "Schedule a job" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#caution-scheduling-system-maintenance", + "title": "Caution: Scheduling system maintenance" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart#invoke-supabase-edge-function-every-30-seconds", + "title": "Invoke Supabase Edge Function every 30 seconds" }, { "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", "title": "Consuming Supabase Queue Messages with Edge Functions" }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", + "title": "Consuming messages in an Edge Function" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", + "title": "Concepts" + }, { "url": "https://supabase.com/docs/guides/queues/api", "title": "API" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", + "title": "pgmq_public.delete(queue_name, message_id)" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", + "title": "pgmq_public.pop(queue_name)" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", + "title": "pgmq_public.send(queue_name, message, sleep_seconds)" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", + "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" - } - ], - "resultChars": 55577 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Cron SQL cron.schedule every minute pgmq.send create queue\", limit: 6) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", + "title": "pgmq_public.archive(queue_name, message_id)" + }, + { + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", + "title": "pgmq_public.read(queue_name, sleep_seconds, n)" + }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-1-enable-extensions", + "title": "Step 1: Enable extensions" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#implementation", + "title": "Implementation" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-architecture", + "title": "Understanding the architecture" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#usage", + "title": "Usage" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#1-create-table-to-store-documents-with-embeddings", + "title": "1. Create table to store documents with embeddings" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#2-create-triggers-to-enqueue-embedding-jobs", + "title": "2. Create triggers to enqueue embedding jobs" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#optional-clearing-embeddings-on-update", + "title": "(Optional) Clearing embeddings on update" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#3-insert-and-update-documents", + "title": "3. Insert and update documents" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#conclusion", + "title": "Conclusion" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#see-also", + "title": "See also" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-visibility-timeouts-work", + "title": "How do visibility timeouts work?" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-we-handle-retries", + "title": "How do we handle retries?" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#is-10-seconds-a-good-interval-for-processing", + "title": "Is 10 seconds a good interval for processing?" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-4-create-the-edge-function", + "title": "Step 4: Create the Edge Function" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-generate-all-embeddings-in-a-single-edge-function-request", + "title": "Why not generate all embeddings in a single Edge Function request?" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-3-create-queue-and-triggers", + "title": "Step 3: Create queue and triggers" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-one-request-per-row", + "title": "Why not one request per row?" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-queue-requests-instead-of-processing-them-immediately", + "title": "Why queue requests instead of processing them immediately?" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-challenge", + "title": "Understanding the challenge" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-2-create-utility-functions", + "title": "Step 2: Create utility functions" } ], - "resultChars": 52683 + "resultChars": 154597 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Queues API delete message_id msg_id pgmq_public rpc delete read create queue SQL\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase Queues JavaScript Edge Function pgmq_public read delete message queue\", limit: 10) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", + "title": "Concepts" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", + "title": "Consuming messages in an Edge Function" }, { "url": "https://supabase.com/docs/guides/queues/api", "title": "API" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", + "title": "pgmq_public.pop(queue_name)" + }, + { + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", + "title": "pgmq_public.delete(queue_name, message_id)" + }, + { + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", + "title": "pgmq_public.read(queue_name, sleep_seconds, n)" + }, + { + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", + "title": "pgmq_public.archive(queue_name, message_id)" + }, + { + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", + "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" + }, + { + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", + "title": "pgmq_public.send(queue_name, message, sleep_seconds)" }, { "url": "https://supabase.com/docs/guides/queues", "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - } - ], - "resultChars": 41311 - } - ] - }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local database restore pg_restore custom format dump existing Postgres migration\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues#features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/queues#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", + "title": "Grant permissions to pgmq_public database functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", + "title": "Pull-Based Queue" + }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/queues/quickstart#message", + "title": "Message" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" + "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", + "title": "Queue types" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" + "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", + "title": "Create Queues" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" + "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", + "title": "What happens when you create a queue?" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - } - ], - "resultChars": 69463 - } - ] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions createClient Authorization header RLS getUser JWT\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", + "title": "Expose Queues to client-side consumers" + }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", + "title": "Enable RLS on your tables in pgmq schema" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", + "title": "Enqueueing and dequeueing messages" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - } - ], - "resultChars": 61836 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a8d67281-e92c-41d8-925e-d0ef86ab9aaa\",\"metric\":\"steps_a_mrpe5p9b\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a8d67281-e92c-41d8-925e-d0ef86ab9aaa\",\"metric\":\"steps_a_mrpe5p9b\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"64cb924b-5c58-4db8-9330-0a9f59eaa773\",\"metric\":\"steps_b_mrpe5p9b\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authentication Authorization header service_role apikey verify JWT getUser Deno serve\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pgmq", + "title": "pgmq: Queues" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#production", + "title": "Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#local-development", + "title": "Local development" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#using-supabase-js", + "title": "Using supabase-js" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#using-a-postgres-client", + "title": "Using a Postgres client" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#using-drizzle", + "title": "Using Drizzle" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres#ssl-connections", + "title": "SSL connections" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#supabase-edge-functions", + "title": "Supabase Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#vercel-edge-functions", + "title": "Vercel Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#manual-configuration", + "title": "Manual configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers#cloudflare-workers", + "title": "Cloudflare Workers" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#sending-messages", + "title": "Sending messages" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#send", + "title": "send" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#send_batch", + "title": "send_batch" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#reading-messages", + "title": "Reading messages" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#read", + "title": "read" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#read_with_poll", + "title": "read_with_poll" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#pop", + "title": "pop" + }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/queues/pgmq#deletingarchiving-messages", + "title": "Deleting/Archiving messages" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/queues/pgmq#delete-single", + "title": "delete (single)" }, { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" + "url": "https://supabase.com/docs/guides/queues/pgmq#delete-batch", + "title": "delete (batch)" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/queues/pgmq#purge_queue", + "title": "purge_queue" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/queues/pgmq#archive-single", + "title": "archive (single)" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - } - ], - "resultChars": 45955 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"config.toml functions function_name entrypoint verify_jwt Edge Functions entrypoint\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#archive-batch", + "title": "archive (batch)" + }, { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" + "url": "https://supabase.com/docs/guides/queues/pgmq#utilities", + "title": "Utilities" }, { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" + "url": "https://supabase.com/docs/guides/queues/pgmq#set_vt", + "title": "set_vt" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/queues/pgmq#list_queues", + "title": "list_queues" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/queues/pgmq#metrics", + "title": "metrics" }, { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" + "url": "https://supabase.com/docs/guides/queues/pgmq#metrics_all", + "title": "metrics_all" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/queues/pgmq#types", + "title": "Types" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/queues/pgmq#message_record", + "title": "message_record" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - } - ], - "resultChars": 52465 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server withSupabase auth user secret service_role Edge Function ctx authMode\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#resources", + "title": "Resources" + }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/queues/pgmq#features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/queues/pgmq#enable-the-extension", + "title": "Enable the extension" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/queues/pgmq#get-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/queues/pgmq#queue-management", + "title": "Queue management" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#create", + "title": "create" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#create_unlogged", + "title": "create_unlogged" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#detach_archive", + "title": "detach_archive" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#drop_queue", + "title": "drop_queue" }, { "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", "title": "Resumable WebSockets with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", - "title": "Deploy MCP servers" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#edge-function-websocket-proxy", + "title": "Edge Function (WebSocket proxy)" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#database-schema", + "title": "Database schema" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#why-this-pattern-works", + "title": "Why this pattern works" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#browser-client", + "title": "Browser client" }, { "url": "https://supabase.com/docs/guides/functions", "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/functions#how-it-works", + "title": "How it works" + }, + { + "url": "https://supabase.com/docs/guides/functions#quick-technical-notes", + "title": "Quick technical notes" + }, + { + "url": "https://supabase.com/docs/guides/functions#when-to-use-edge-functions", + "title": "When to use Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions#examples", + "title": "Examples" } ], - "resultChars": 70886 + "resultChars": 252301 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Authorization headers Edge Functions legacy service_role key apikey secret mode @supabase/server\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Queues API Reference create queue send read delete SQL pgmq.create\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/queues/pgmq#drop_queue", + "title": "drop_queue" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/queues/pgmq#deletingarchiving-messages", + "title": "Deleting/Archiving messages" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/queues/pgmq#delete-single", + "title": "delete (single)" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/queues/pgmq#delete-batch", + "title": "delete (batch)" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - } - ], - "resultChars": 59522 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019f71be-cbd8-77ae-b3e9-267b90ee4ff8/receipt-alpha.pdf, 019f71be-cbd8-77ae-b3e9-267b90ee4ff8/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets all requirements: private user-files bucket, owner-scoped authenticated SELECT and INSERT RLS policies on storage.objects with WITH CHECK for uploads, RLS not disabled, and supabase-js createSignedUrl with expiry for temporary sharing." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage access control folder name auth.uid signed URL createSignedUrl private bucket RLS policy\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#purge_queue", + "title": "purge_queue" + }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/queues/pgmq#archive-single", + "title": "archive (single)" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#archive-batch", + "title": "archive (batch)" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#utilities", + "title": "Utilities" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#set_vt", + "title": "set_vt" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#list_queues", + "title": "list_queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq#metrics", + "title": "metrics" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/queues/pgmq#metrics_all", + "title": "metrics_all" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/queues/pgmq#types", + "title": "Types" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/queues/pgmq#message_record", + "title": "message_record" }, { - "url": "https://supabase.com/docs/guides/storage/s3/authentication", - "title": "S3 Authentication" - } - ], - "resultChars": 18366 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createSignedUrl storage expiresIn example signedUrl supabase-js\", limit: 3) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/queues/pgmq#resources", + "title": "Resources" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/queues/pgmq#features", + "title": "Features" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - } - ], - "resultChars": 6141 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "5 passed, 3 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly concludes that `posts` has the tenant isolation flaw, specifically cross-tenant post visibility for authenticated members, and grounds it in the pgTAP failures. It also correctly distinguishes `notes` as passing." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database testing pgTAP RLS auth.uid set request.jwt.claims\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#enable-the-extension", + "title": "Enable the extension" + }, { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" + "url": "https://supabase.com/docs/guides/queues/pgmq#get-usage", + "title": "Usage " }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/queues/pgmq#queue-management", + "title": "Queue management" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/queues/pgmq#create", + "title": "create" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/queues/pgmq#create_unlogged", + "title": "create_unlogged" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 89166 - } - ] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search match_documents vector(384) RLS security invoker RPC\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#detach_archive", + "title": "detach_archive" + }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/queues/pgmq#sending-messages", + "title": "Sending messages" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/queues/pgmq#send", + "title": "send" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/queues/pgmq#send_batch", + "title": "send_batch" }, { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/queues/pgmq#reading-messages", + "title": "Reading messages" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - } - ], - "resultChars": 53178 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "low" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: preserves app scrape, adds HTTPS Supabase Metrics API scrape at the correct path and project host, uses basic_auth with password_file, and wires the password file via a Docker Compose secret." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase Secret API key, writing it to the expected secret file, starting/recreating the Compose stack, and verifying via curl, Prometheus targets, and Grafana/PromQL." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics endpoint customer v1 privileged metrics service_role basic auth Grafana\", limit: 8) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/queues/pgmq#read", + "title": "read" + }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/queues/pgmq#read_with_poll", + "title": "read_with_poll" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#5-operating-tips", - "title": "5. Operating tips" + "url": "https://supabase.com/docs/guides/queues/pgmq#pop", + "title": "pop" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#4-configure-alerting", - "title": "4. Configure alerting" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#3-import-supabase-dashboards", - "title": "3. Import Supabase dashboards" + "url": "https://supabase.com/docs/guides/queues#resources", + "title": "Resources" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#2-deploy-grafana", - "title": "2. Deploy Grafana" + "url": "https://supabase.com/docs/guides/queues#features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#1-deploy-prometheus", - "title": "1. Deploy Prometheus" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/queues/quickstart#create-queues", + "title": "Create Queues" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/queues/quickstart#message", + "title": "Message" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#5-multi-project-setups", - "title": "5. Multi-project setups" + "url": "https://supabase.com/docs/guides/queues/quickstart#queue-types", + "title": "Queue types" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#components", - "title": "Components" + "url": "https://supabase.com/docs/guides/queues/quickstart#what-happens-when-you-create-a-queue", + "title": "What happens when you create a queue?" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#1-define-the-scrape-job", - "title": "1. Define the scrape job" + "url": "https://supabase.com/docs/guides/queues/quickstart#expose-queues-to-client-side-consumers", + "title": "Expose Queues to client-side consumers" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#collector-specific-notes", - "title": "Collector-specific notes" + "url": "https://supabase.com/docs/guides/queues/quickstart#enable-rls-on-your-tables-in-pgmq-schema", + "title": "Enable RLS on your tables in pgmq schema" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#2-secure-the-credentials", - "title": "2. Secure the credentials" + "url": "https://supabase.com/docs/guides/queues/quickstart#grant-permissions-to-pgmq_public-database-functions", + "title": "Grant permissions to pgmq_public database functions" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#3-downstream-dashboards", - "title": "3. Downstream dashboards" + "url": "https://supabase.com/docs/guides/queues/quickstart#enqueueing-and-dequeueing-messages", + "title": "Enqueueing and dequeueing messages" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#4-alerts-and-automation", - "title": "4. Alerts and automation" + "url": "https://supabase.com/docs/guides/queues/quickstart#concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/queues/quickstart#pull-based-queue", + "title": "Pull-Based Queue" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#prerequisites", - "title": "Prerequisites" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#1-create-a-grafana-cloud-stack", - "title": "1. Create a Grafana Cloud stack" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicreadqueue_name-sleep_seconds-n", + "title": "pgmq_public.read(queue_name, sleep_seconds, n)" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#2-configure-the-supabase-integration", - "title": "2. Configure the Supabase integration" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicdeletequeue_name-message_id", + "title": "pgmq_public.delete(queue_name, message_id)" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#5-troubleshooting", - "title": "5. Troubleshooting" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicarchivequeue_name-message_id", + "title": "pgmq_public.archive(queue_name, message_id)" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#4-configure-alerts-optional", - "title": "4. Configure alerts (optional)" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsend_batchqueue_name-messages-sleep_seconds", + "title": "pgmq_public.send_batch(queue_name, messages, sleep_seconds)" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#3-import-the-supabase-dashboard", - "title": "3. Import the Supabase dashboard" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicsendqueue_name-message-sleep_seconds", + "title": "pgmq_public.send(queue_name, message, sleep_seconds)" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/queues/api#pgmq_publicpopqueue_name", + "title": "pgmq_public.pop(queue_name)" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics#what-you-can-do-with-the-metrics-api", - "title": "What you can do with the Metrics API" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics#choose-your-monitoring-stack", - "title": "Choose your monitoring stack" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#concepts", + "title": "Concepts" }, { - "url": "https://supabase.com/docs/guides/telemetry/metrics#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions#consuming-messages-in-an-edge-function", + "title": "Consuming messages in an Edge Function" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/grafana-not-displaying-data-sXJrMj", - "title": "Grafana not displaying data" + "url": "https://supabase.com/docs/guides/database/extensions/pgmq", + "title": "pgmq: Queues" } ], - "resultChars": 128863 + "resultChars": 240794 } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-database-001-prometheus-metrics.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -16332,34 +14487,36 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "low" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", "product": [ - "edge-functions" + "database" ], "topic": [ - "security" + "migrations" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", + "name": "all 3 tables exist (teams, members, tasks)", "passed": true }, { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "name": "foreign key constraints survived the restore", + "passed": true }, { - "name": "WEATHER_API_KEY value is not committed to the repo", + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", "passed": true } ], @@ -16371,50 +14528,126 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions deploy secrets set env-file WEATHER_API_KEY Deno.env.get verify_jwt config.toml\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI local development restore pg_dump custom format local database\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + } + ], + "resultChars": 42818 + } + ] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient auth.getUser RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/functions/examples/cloudflare-turnstile", - "title": "CAPTCHA support with Cloudflare Turnstile" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/functions/examples/slack-bot-mention", - "title": "Slack Bot Mention Edge Function" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" } ], - "resultChars": 63993 + "resultChars": 36221 } ] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -16425,35 +14658,67 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "low" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", "product": [ - "database", + "edge-functions", "auth", - "storage" + "database" ], "topic": [ - "self-hosting" + "sdk", + "rls", + "security" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "secrets rotated off the shipped defaults", - "passed": true + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"3e84c2e6-24fe-4fff-b245-cc723cf23fbb\",\"metric\":\"steps_a_mry6lgai\",\"value\":111}]" }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"3e84c2e6-24fe-4fff-b245-cc723cf23fbb\",\"metric\":\"steps_a_mry6lgai\",\"value\":111}]" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"32b74861-1f4b-418e-90b7-d7583076fb60\",\"metric\":\"steps_b_mry6lgai\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -16464,723 +14729,912 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env secrets JWT keys current setup\", limit: 8) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Edge Functions JWT verification verify_jwt false getUser Authorization header service role key apikey secret key environment variables\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", - "title": "Contents" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", - "title": "System requirements" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", - "title": "Installing Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", - "title": "Quick start (Linux)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", - "title": "Manual installation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", - "title": "Configuring and securing Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", - "title": "Generate keys and secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", - "title": "Configure Supabase URLs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", - "title": "Where to find your credentials" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", - "title": "Studio authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", - "title": "Starting and stopping" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", - "title": "Accessing Supabase Studio (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", - "title": "Accessing Postgres" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", - "title": "Accessing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", - "title": "Accessing APIs" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", - "title": "Enabling analytics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", - "title": "Configuring HTTPS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", - "title": "Managing the stack" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", - "title": "Updating" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", - "title": "Uninstalling" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", - "title": "Advanced topics" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", - "title": "Architecture" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", - "title": "Setting database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", - "title": "Configuring phone login, SMS, and MFA" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", - "title": "Accessing Postgres through Supavisor" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", - "title": "Kong API gateway routing" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", - "title": "Differences from the Supabase platform" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", - "title": "Backward compatibility" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", - "title": "Rotating the new API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", - "title": "Regenerating asymmetric key pair" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", - "title": "How it works" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", - "title": "What client SDK sends" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", - "title": "Additional resources" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-environment-variables", - "title": "Auth environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-by-step-configuration", - "title": "Step-by-step configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-1-register-your-app-with-the-provider", - "title": "Step 1: Register your app with the provider" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-2-configure-environment-variables", - "title": "Step 2: Configure environment variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-3-enable-the-matching-lines-in-docker-compose-configuration", - "title": "Step 3: Enable the matching lines in Docker Compose configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-4-restart-the-auth-service", - "title": "Step 4: Restart the auth service" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-5-verify-the-configuration", - "title": "Step 5: Verify the configuration" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-specific-setup", - "title": "Provider-specific setup" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#other-supported-providers", - "title": "Other supported providers" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#environment-variable-reference", - "title": "Environment variable reference" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-service-fails-to-start", - "title": "Auth service fails to start" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#nonce-check-failure-on-mobile-google-sign-in", - "title": "Nonce check failure on mobile (Google Sign In)" - }, + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + } + ], + "resultChars": 65864 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Securing Edge Functions auth getUser token Authorization createClient RLS\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#site-url-or-redirect-url-errors-after-login", - "title": "Site URL or redirect URL errors after login" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#variables-added-to-the-environment-but-provider-still-not-working", - "title": "Variables added to the environment but provider still not working" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#test-the-login-flow", - "title": "Test the login flow" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#oauth-request-flow", - "title": "OAuth request flow" - }, + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + } + ], + "resultChars": 65939 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"@supabase/server verifyApiKey SUPABASE_SECRET_KEYS Edge Functions\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" - }, + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + } + ], + "resultChars": 101795 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"@supabase/server legacy service_role key apikey auth mode service role\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" - }, + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 119148 + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019f9166-2833-711a-b6e5-de727d275f06/receipt-alpha.pdf, 019f9166-2833-711a-b6e5-de727d275f06/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets requirements: private user-files bucket, RLS remains enabled, authenticated SELECT and INSERT policies scoped to bucket and user's UID folder, and supabase-js createSignedUrl with short expiry for temporary sharing." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Storage access control RLS policy foldername auth.uid signed URL createSignedUrl supabase-js private bucket\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-the-success-screen", - "title": "Create the success screen" - }, + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + } + ], + "resultChars": 19742 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript createSignedUrl path expiresIn download option storage-from-createsignedurl\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-the-mainactivity", - "title": "Implement the MainActivity" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-screens", - "title": "Implement screens" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-repositories", - "title": "Implement repositories" - }, + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + } + ], + "resultChars": 2650 + } + ] + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/tenant_isolation.test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "4 passed, 2 failed" + }, + { + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "The agent correctly identifies `posts` as having a tenant isolation flaw, explicitly says `notes` is correctly scoped, and grounds the conclusion in pgTAP results showing the cross-tenant `posts` leak." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI database testing pgTAP RLS auth.uid tenant isolation tests\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-a-data-transfer-object", - "title": "Create a data transfer object" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#provide-supabase-instances-with-hilt", - "title": "Provide Supabase instances with Hilt" + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-hilt-for-dependency-injection", - "title": "Set up Hilt for dependency injection" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-supabase-dependencies", - "title": "Set up Supabase dependencies" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#use-value-from-buildconfig", - "title": "Use value from BuildConfig" - }, + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" + } + ], + "resultChars": 71814 + } + ] + }, + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pgvector gte-small 384 dimensions semantic search match_documents row level security rpc auth uid\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#read-and-set-value-to-buildconfig", - "title": "Read and set value to BuildConfig" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-local-environment-secret", - "title": "Create local environment secret" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-api-key-and-secret-securely", - "title": "Set up API key and secret securely" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-new-android-project", - "title": "Create new Android project" + "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", + "title": "Choosing your Compute Add-on" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#building-the-app", - "title": "Building the app" - }, + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" + } + ], + "resultChars": 102613 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Meets requirements: app scrape preserved; Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, supabase.co project target template, and Compose mounts the password file via a secret." + }, + { + "name": "documented live deployment and verification steps", + "passed": true, + "judgeNotes": "README includes Secret API key creation, matching Docker secret file placement, Compose start/recreate steps, and concrete verification via Prometheus targets and PromQL/Grafana." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Prometheus metrics endpoint project metrics observability authentication service role Grafana\", limit: 8) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-google-authentication", - "title": "Set up Google authentication" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#get-api-details", - "title": "Get API details" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#4-configure-alerting", + "title": "4. Configure alerting" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-the-database-schema", - "title": "Set up the database schema" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#3-import-supabase-dashboards", + "title": "3. Import Supabase dashboards" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-a-project", - "title": "Create a project" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#2-deploy-grafana", + "title": "2. Deploy Grafana" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#project-setup", - "title": "Project setup" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#1-deploy-prometheus", + "title": "1. Deploy Prometheus" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#architecture", + "title": "Architecture" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", - "title": "Custom environment variables" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-self-hosted#5-operating-tips", + "title": "5. Operating tips" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", - "title": "Using an env file (recommended)" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", - "title": "Using inline environment variables" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#1-create-a-grafana-cloud-stack", + "title": "1. Create a Grafana Cloud stack" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", - "title": "Accessing variables in functions" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#prerequisites", + "title": "Prerequisites" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", - "title": "Calling Supabase services from functions" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#2-configure-the-supabase-integration", + "title": "2. Configure the Supabase integration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", - "title": "Internal vs external URLs" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#3-import-the-supabase-dashboard", + "title": "3. Import the Supabase dashboard" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", - "title": "Managing functions via dashboard" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#4-configure-alerts-optional", + "title": "4. Configure alerts (optional)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", - "title": "Deploying functions to a remote server" + "url": "https://supabase.com/docs/guides/telemetry/metrics/grafana-cloud#5-troubleshooting", + "title": "5. Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", - "title": "Memory or timeout errors" + "url": "https://supabase.com/docs/guides/telemetry/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", - "title": "Custom env vars not available in functions" + "url": "https://supabase.com/docs/guides/telemetry/metrics#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", - "title": "Changes to function code not reflected after editing" + "url": "https://supabase.com/docs/guides/telemetry/metrics#choose-your-monitoring-stack", + "title": "Choose your monitoring stack" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", - "title": "Copying functions from Supabase platform" + "url": "https://supabase.com/docs/guides/telemetry/metrics#what-you-can-do-with-the-metrics-api", + "title": "What you can do with the Metrics API" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", - "title": "500 error on invocation" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#collector-specific-notes", + "title": "Collector-specific notes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", - "title": "Invoke the default function" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#2-secure-the-credentials", + "title": "2. Secure the credentials" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", - "title": "Create a new function" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#3-downstream-dashboards", + "title": "3. Downstream dashboards" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", - "title": "Step 1: Add a new function directory and the function code" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#4-alerts-and-automation", + "title": "4. Alerts and automation" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", - "title": "Step 2: Restart the functions service to pick up the new function" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#5-multi-project-setups", + "title": "5. Multi-project setups" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", - "title": "Step 3: Invoke your function" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#components", + "title": "Components" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" + "url": "https://supabase.com/docs/guides/telemetry/metrics/vendor-agnostic#1-define-the-scrape-job", + "title": "1. Define the scrape job" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#postgres-17-fails-to-start-with-a-leftover-db-config-volume", - "title": "Postgres 17 fails to start with a leftover db-config volume" + "url": "https://supabase.com/docs/guides/database/connection-management", + "title": "Connection management" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#restoring-from-a-manual-backup", - "title": "Restoring from a manual backup" + "url": "https://supabase.com/docs/guides/database/connection-management#capturing-historical-usage", + "title": "Capturing historical usage" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/database/connection-management#monitoring-connections", + "title": "Monitoring connections" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#create-a-backup", - "title": "Create a backup" + "url": "https://supabase.com/docs/guides/database/connection-management#configuring-supavisors-pool-size", + "title": "Configuring Supavisor's pool size" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#what-the-upgrade-does", - "title": "What the upgrade does" + "url": "https://supabase.com/docs/guides/database/connection-management#connections", + "title": "Connections" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-an-existing-postgres-15-deployment", - "title": "Upgrade an existing Postgres 15 deployment" + "url": "https://supabase.com/docs/guides/database/connection-management#observing-live-connections", + "title": "Observing live connections" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#extensions-removed-in-postgres-17", - "title": "Extensions removed in Postgres 17" + "url": "https://supabase.com/docs/guides/database/connection-management#grafana-dashboard", + "title": "Grafana Dashboard" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#run-the-upgrade", - "title": "Run the upgrade" + "url": "https://supabase.com/docs/guides/database/connection-management#dashboard-monitoring-charts", + "title": "Dashboard monitoring charts" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#after-the-upgrade", - "title": "After the upgrade" + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#rollback", - "title": "Rollback" + "url": "https://supabase.com/docs/guides/platform/read-replicas#dedicated-connection-pool", + "title": "Dedicated connection pool" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#custom-postgres-configuration", - "title": "Custom Postgres configuration" + "url": "https://supabase.com/docs/guides/platform/read-replicas#dedicated-endpoints", + "title": "Dedicated endpoints" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-process-details", - "title": "Upgrade process details" + "url": "https://supabase.com/docs/guides/platform/read-replicas#features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/platform/read-replicas#about-read-replicas", + "title": "About Read Replicas" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pg_upgrade-fails-with-replication-slot-errors", - "title": "pg_upgrade fails with replication slot errors" + "url": "https://supabase.com/docs/guides/platform/read-replicas#pricing", + "title": "Pricing" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pgsodium--supabase-vault-errors", - "title": "pgsodium / Supabase Vault errors" + "url": "https://supabase.com/docs/guides/platform/read-replicas#centralized-configuration-management", + "title": "Centralized configuration management" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#services-fail-to-connect-after-upgrade", - "title": "Services fail to connect after upgrade" + "url": "https://supabase.com/docs/guides/platform/read-replicas#metrics", + "title": "Metrics" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#disk-space-issues-during-upgrade", - "title": "Disk space issues during upgrade" + "url": "https://supabase.com/docs/guides/platform/read-replicas#logging", + "title": "Logging" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#new-deployment-with-postgres-17", - "title": "New deployment with Postgres 17" + "url": "https://supabase.com/docs/guides/platform/read-replicas#querying-through-the-sql-editor", + "title": "Querying through the SQL editor" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#before-you-begin", - "title": "Before you begin" - }, + "url": "https://supabase.com/docs/guides/platform/read-replicas#api-load-balancer", + "title": "API load balancer" + } + ], + "resultChars": 151336 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment via Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy supabase functions deploy secrets set env file CORS\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-4-restore-to-your-self-hosted-database", - "title": "Step 4: Restore to your self-hosted database" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-3-prepare-your-self-hosted-instance", - "title": "Step 3: Prepare your self-hosted instance" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-2-back-up-your-platform-database", - "title": "Step 2: Back up your platform database" + "url": "https://supabase.com/docs/guides/functions/storage-caching", + "title": "Integrating with Supabase Storage" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-1-get-your-platform-connection-string", - "title": "Step 1: Get your platform connection string" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#whats-included-in-the-restore-and-whats-not", - "title": "What's included in the restore and what's not" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-5-verify-the-restore", - "title": "Step 5: Verify the restore" - }, + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + } + ], + "resultChars": 84017 + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"self-hosting Docker compose docker .env JWT secret anon key service role key secrets SMTP S3 2026\", limit: 8) { nodes { __typename title href content } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#auth-considerations", - "title": "Auth considerations" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#postgres-version-compatibility", - "title": "Postgres version compatibility" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#version-mismatches-between-platform-and-self-hosted", - "title": "Version mismatches between platform and self-hosted" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#extension-not-available", - "title": "Extension not available" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#connection-refused", - "title": "Connection refused" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", + "title": "Custom Email Templates" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#legacy-studio-configuration", - "title": "Legacy Studio configuration" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", + "title": "Configure Phone Login & MFA" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#custom-roles-missing-passwords", - "title": "Custom roles missing passwords" - }, + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + } + ], + "resultChars": 155205 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Self-hosting with Docker quick start clone repository copy docker .env generate keys minimum requirements reverse proxy\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-not-enabled-or-provider-seen-as-false-in-settings" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#401-invalid-jwt" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#permission-denied-on-the-data-directory" + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" } ], - "resultChars": 592856 + "resultChars": 89048 } ] }, @@ -17239,7 +15693,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements real auth user deletion with cascading sessions/identities and RLS active-user checks, and explains the remaining JWT caveat consistently: Data API paths covered by its RLS check are blocked after commit, but purely local validation or other services/policies may accept the JWT until expiry. It also correctly distinguishes publishable frontend/RLS use from secret server-only RLS-bypassing use." + "judgeNotes": "Meets the core requirements: diagnoses soft-delete-only flow, deletes auth user/sessions via cascade, adds RLS guard closing Data API stale-JWT access, explains JWTs remain valid until expiry, and correctly distinguishes publishable vs secret keys." } ], "skills": { @@ -17250,131 +15704,127 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user JWT remains valid until expiry revoke sessions access token RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"delete user access token remains valid until expiry revoke sessions JWT RLS API keys publishable secret legacy anon service_role\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, { "url": "https://supabase.com/docs/guides/auth/sessions", "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", + "title": "Rotating Anon, Service, and JWT Secrets" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" } ], - "resultChars": 40980 + "resultChars": 120260 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key anon service_role RLS new API keys\", limit: 6) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ManagementApiReference { title href content } } } }", + "query": "query { searchDocs(query: \"Supabase sessions sign out access token valid until expiry delete user\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/auth/signout", + "title": "Signing out" }, { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" } ], - "resultChars": 197013 + "resultChars": 70080 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret key bypass RLS Authorization header apikey\", limit: 10) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"delete user auth.users database delete user JWT remains valid Supabase\", limit: 6) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/storage/debugging/error-codes", - "title": "Error Codes" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" } ], - "resultChars": 174374 + "resultChars": 66952 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"secret key RLS user Authorization JWT publishable key authenticated role\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"publishable key frontend secret key RLS anon authenticated service_role bypass RLS\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { "url": "https://supabase.com/docs/guides/database/secure-data", "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" } ], - "resultChars": 56289 + "resultChars": 61666 } ] }, @@ -17429,7 +15879,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Diagnosed orders missing from supabase_realtime despite SUBSCRIBED channel, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified courier_locations remained and did not alter RLS/policies." + "judgeNotes": "The assistant correctly identified orders missing from supabase_realtime as the root cause, added only public.orders to the existing publication, verified courier_locations remained, and did not weaken RLS or policies." } ], "skills": { @@ -17437,7 +15887,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication ALTER PUBLICATION\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines", + "title": "Set up Pipelines" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + } + ], + "resultChars": 121815 + } + ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", @@ -17463,22 +15942,22 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant named image-transform as affected and described eight intermittent HTTP 503s across 07:00–12:00 UTC on 2026-04-28, matching the required recurring pattern." + "judgeNotes": "The assistant explicitly identified `image-transform` as affected and described eight intermittent 503 responses from 07:00–12:00 UTC on 2026-04-28, matching the required recurring pattern." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although it correctly notes the 503s had gateway entries with no function executions and distinguishes them from avatar-upload's executed 500, it also suggests pinning the image-transform NPM dependency and redeploying, implying a function/startup dependency remediation rather than clearly attributing the recurring 503s to the gateway/platform layer in front of the function." + "passed": true, + "judgeNotes": "Attributes image-transform 503s to pre-handler gateway/platform/runtime-startup layer, grounded in absence from execution logs and unchanged deployment with successful retries; also distinguishes gateway 503s from avatar-upload's function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete actionable next steps, including classifying 503 metadata, redeploying with pinned/bundled dependency, adding retries, correlating timestamps with status history, and opening a support case with request IDs." + "judgeNotes": "Recommended a concrete next step: opening a Supabase support case with specific timestamps/request IDs, plus targeted follow-up actions." } ], "skills": { @@ -17486,36 +15965,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions 503 error function not invoked gateway troubleshooting retry\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", - "title": "Edge Function 404 error response" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 40284 - } - ] + "calls": [] }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", @@ -17573,7 +16023,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all for the Data API, kept RLS enabled, and created authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK. No permissive/public/anon policy or RLS disabling was used." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() = user_id with WITH CHECK for insert." } ], "skills": { @@ -17584,23 +16034,27 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security auth.uid select insert policy authenticated\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security auth.uid select insert policy authenticated users own rows\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, { "url": "https://supabase.com/docs/guides/getting-started/features", "title": "Features" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", - "title": "Storage error: 403 Forbidden: 'new row violates row-level security policy' on upload" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" } ], - "resultChars": 42195 + "resultChars": 68712 } ] }, @@ -17653,7 +16107,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied by `supabase db push --db-url \"$DB_URL\"` in action #9. The orphan bio migration was reconciled by adding `supabase/migrations/20240115000000_add_profile_bio.sql` locally in action #8, then pushing via the CLI. Only read-only `psql` inspection was used; no direct SQL mutation or prepared-statement workaround was seen." + "judgeNotes": "Avatar migration was applied by the real `supabase db push --db-url \"$DB_URL\"` in action #9, followed by verification showing `20240220000000 | add_avatar_url` and the `avatar_url` column. History was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` in #8, then pushing with Supabase CLI in #9. No prohibited direct SQL mutation or migration-history edit was used; read-only `psql SELECT` checks were present. Workaround seen: using the IPv4 pooler `--db-url` instead of linked project hostname." } ], "skills": { @@ -17664,17 +16118,9 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history repair duplicate column remote migration list\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI migration up db push repair migration history remote hosted project\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, { "url": "https://supabase.com/docs/reference/cli/supabase-db-push", "title": "Push new migrations to the remote database" @@ -17684,11 +16130,19 @@ "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-list", - "title": "List local and remote migrations" + "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", + "title": "Repair the migration history table" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" } ], - "resultChars": 50099 + "resultChars": 61657 } ] }, diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index c61b8588..03050f21 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -33,22 +33,22 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"msg\":\"Error: Missing authorization header\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"5f03d0cf-712b-4853-b2e5-5dc58fb7885c\",\"metric\":\"steps_a_mrxmjocl\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"50d1e032-65f2-4098-9ddb-2ebcda506556\",\"metric\":\"steps_a_mry6dogx\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"5f03d0cf-712b-4853-b2e5-5dc58fb7885c\",\"metric\":\"steps_a_mrxmjocl\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"50d1e032-65f2-4098-9ddb-2ebcda506556\",\"metric\":\"steps_a_mry6dogx\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"8a961f91-1a2b-4618-8fc7-cf6582bc40f6\",\"metric\":\"steps_b_mrxmjocl\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"92f72a05-208b-494e-af1e-5b9f267757bb\",\"metric\":\"steps_b_mry6dogx\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -58,7 +58,7 @@ { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"msg\":\"Invalid JWT\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", @@ -84,17 +84,13 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server edge function withSupabase multi-auth\", limit: 10) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"@supabase/server withSupabase edge function auth\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", "title": "Which package to use" }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" @@ -104,98 +100,35 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", "title": "Resumable WebSockets with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/astrojs", - "title": "Use Supabase Auth with Astro" - } - ], - "resultChars": 12 - }, - { - "source": "web_search", - "query": "\"@supabase/server\" npm withSupabase edge function", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/blog/introducing-supabase-server", - "title": "Introducing @supabase/server" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/routing", - "title": "Handling Routing in Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/blog/edge-functions-node-npm", - "title": "Edge Functions: Node and native npm compatibility" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres) | Supabase Docs" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai-tools/ai-prompts/edge-functions", - "title": "AI Prompt: Writing Supabase Edge Functions | Supabase Docs" + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions | Supabase Docs" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth | Supabase Docs" - } - ], - "resultChars": 2932 - }, - { - "source": "web_fetch", - "query": "Extract full details on the withSupabase function: its signature, options (auth modes like publishable/secret), the ctx object shape (supabase, supabaseAdmin, user, etc), how it determines auth mode from headers (Authorization vs apikey), how user_id is passed for service-role calls, and any code examples for edge functions. Include exact code snippets verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/blog/introducing-supabase-server" - } - ], - "resultChars": 1850 - }, - { - "source": "web_search", - "query": "supabase functions serve \"failed to determine entrypoint\" edge-runtime", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/troubleshooting", - "title": "Supabase Docs | Edge Functions Troubleshooting" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-fails-deploy", - "title": "Supabase Docs | Troubleshooting | Edge Function deploy failed due to an internal error" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" } ], - "resultChars": 2736 + "resultChars": 12 } ] }, @@ -233,7 +166,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The answer correctly treats the issue as Supabase Realtime/Postgres Changes setup, adds the table to the supabase_realtime publication, verifies it, and gives a postgres_changes client subscription. It does not recommend or imply read replicas are needed or useful." + "judgeNotes": "The assistant correctly treated the issue as a Supabase Realtime/Postgres Changes setup, added the table to the supabase_realtime publication, verified it, and provided client postgres_changes subscription code. It did not recommend or imply read replicas are needed/useful for live updates, nor confuse them with logical replication/publications." } ], "skills": { @@ -246,36 +179,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"enable realtime for a table postgres changes\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" - }, - { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" - } - ], - "resultChars": 150974 - } - ] + "calls": [] }, "prompt": "I'm building a simple chat app on Supabase.\n\nUsers can send messages, and I want everyone in the same room to see new\nmessages appear automatically without refreshing the page.\n\nCan you inspect the project and set up whatever Supabase needs for live updates?", "promptSourcePath": "evals/build-realtime-001-live-chat-updates/PROMPT.md", @@ -305,17 +209,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "The assistant correctly identified `video-thumbnails` as the affected function and explicitly stated the failures are `status_code: 546` due to CPU/resource limits, while ruling out the unrelated `welcome-email` 500." + "judgeNotes": "Identified video-thumbnails as the affected function and correctly recognized HTTP 546/resource limit CPUTime errors rather than 500/503 server errors." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion and grounds it in log evidence: shutdown reason CPUTime plus cpu_time_used 2000ms at cpu_time_limit 2000ms. It also explicitly rules out unrelated errors." + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence: shutdown reason CPUTime and cpu_time_used 2000ms matching cpu_time_limit 2000ms. It also distinguishes this from wall-clock time and identifies the CPU-bound thumbnail generation as the likely cause." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant correctly identifies CPUTime limit as root cause and recommends reducing/offloading CPU-intensive thumbnail generation: rejecting/downscaling large inputs, moving work to a background/external worker, and optimizing/profiling the thumbnailing code. It also notes the CPU ceiling cannot be raised." + "judgeNotes": "The assistant identified CPUTime limit as the root cause and recommended reducing/offloading CPU-intensive thumbnail generation, including moving video decoding to a background/external worker, reducing per-call CPU cost, and routing large inputs asynchronously." } ], "skills": { @@ -402,7 +306,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "Diagnosed missing Data API/table grants separately from RLS, preserved owner-scoped RLS, granted only SELECT/INSERT on public.journal_entries to authenticated, did not grant anon/public or weaken/disable RLS, and verified behavior." + "judgeNotes": "Diagnoses missing Data API/table GRANTs under secure-default behavior, distinguishes GRANTs from RLS, preserves owner-scoped RLS, and grants only SELECT/INSERT on public.journal_entries to authenticated (not anon/public)." } ], "skills": { @@ -419,7 +323,7 @@ }, "prompt": "Our app lets signed-in users keep a private journal. Entries are stored in the `journal_entries` table. A user should only ever see their own, and they need to be able to create new entries from the app.\n\nI can see the rows when I query the table directly, but the journal page shows an empty list for every signed-in user. New entries don't show up after saving either. Please figure out why and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-secure-default-grants/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-dataapi-002-secure-default-grants.json" }, { @@ -450,7 +354,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"8326f375-f2b1-45eb-9a76-8484d1f56545\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"34b271f9-c0f6-441a-85ab-c4b08b047ace\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -459,7 +363,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "Diagnosed the zero-row UPDATE as caused by the tasks UPDATE RLS policy missing a USING clause while only having WITH CHECK, explained the USING vs WITH CHECK behavior, and fixed it with a policy scoped to authenticated users using (user_id = auth.uid()) while preserving with check (user_id = auth.uid()) and keeping RLS intact." + "judgeNotes": "The answer correctly identifies the missing USING clause on the authenticated UPDATE policy as the cause of silent 0-row updates, explains WITH CHECK vs USING, and fixes it with `USING (user_id = auth.uid())` while preserving `WITH CHECK (user_id = auth.uid())`, scoped to authenticated users and not disabling RLS." } ], "skills": { @@ -472,36 +376,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"row level security policy update missing USING clause WITH CHECK\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0024_permissive_rls_policy", - "title": "Database Advisor: Lint 0024_permissive_rls_policy" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", - "title": "RLS Simplified" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0007_policy_exists_rls_disabled", - "title": "Database Advisor: Lint 0007_policy_exists_rls_disabled" - } - ], - "resultChars": 59228 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", @@ -531,7 +406,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer gives safe, docs-grounded recovery guidance: it explicitly says pause/restore is not the right step, recommends restart first, distinguishes restart from heavier backup/reactivation restore behavior, and suggests checking resource pressure and contacting support if unresolved." + "judgeNotes": "The answer correctly rejects pause/restore as the first recovery step, distinguishes restart from pause/restore, and recommends restarting the database plus checking resource sizing/performance if recurrence occurs. It is reasonably docs-grounded and safe, though it could be more complete by explicitly mentioning logs/advisors, reducing workload/scaling, unhealthy-services troubleshooting, waiting a few minutes, and contacting Supabase support if unresolved." } ], "skills": { @@ -547,59 +422,159 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"restart project unhealthy pause restore\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"project unhealthy restart pause restore dashboard unusable\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } } ", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#types-of-backups", + "title": "Types of backups" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#backup-and-restore-process", + "title": "Backup and restore process" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#managing-backups-programmatically", + "title": "Managing backups programmatically" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#restoration-process", + "title": "Restoration process" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#point-in-time-recovery", + "title": "Point-in-Time recovery" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#backup-process", + "title": "Backup process" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#restoration-process-1", + "title": "Restoration process" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#pricing", + "title": "Pricing" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#downloading-backups-after-disabling-pitr", + "title": "Downloading backups after disabling PITR" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups#restore-to-a-new-project", + "title": "Restore to a new project" + }, { "url": "https://supabase.com/docs/guides/platform/upgrading", "title": "Upgrading" }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#breaking-changes", + "title": "Breaking changes" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#in-place-upgrades", + "title": "In-place upgrades" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#pause-and-restore", + "title": "Pause and restore" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#caveats", + "title": "Caveats" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#logical-replication", + "title": "Logical replication" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#time-limits", + "title": "Time limits" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#disk-sizing", + "title": "Disk sizing" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#objects-dependent-on-postgres-extensions", + "title": "Objects dependent on Postgres extensions" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#pg_cron-records", + "title": "pg_cron records" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#extensions", + "title": "Extensions" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#authentication-method-changes---deprecating-md5-in-favor-of-scram-sha-256", + "title": "Authentication method changes - deprecating md5 in favor of scram-sha-256" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#database-size-reduction", + "title": "Database size reduction" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#post-upgrade-validation", + "title": "Post-upgrade validation" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#specific-upgrade-notes", + "title": "Specific upgrade notes" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#upgrading-to-postgres-17", + "title": "Upgrading to Postgres 17" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading#upgrading-to-pg_graphql-160", + "title": "Upgrading to pg_graphql 1.6.0" + }, { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/platform/delete-project", - "title": "Deleting Your Project" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing#preventing-automatic-project-pausing", + "title": "Preventing automatic project pausing" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing#how-automatic-pausing-works", + "title": "How automatic pausing works" }, { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - } - ], - "resultChars": 32704 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"project unhealthy restart dashboard unresponsive troubleshooting\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing#restoring-a-paused-project", + "title": "Restoring a paused project" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", - "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + "url": "https://supabase.com/docs/guides/platform/free-project-pausing#90-day-window-to-restore", + "title": "90-day window to restore" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", - "title": "Supabase dashboard not loading/ Project not loading on dashboard" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup#downloading-your-backup", + "title": "Downloading your backup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup#restoring-your-backup", + "title": "Restoring your backup" } ], - "resultChars": 34604 + "resultChars": 72750 } ] }, @@ -649,7 +624,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019f8f6b-0da1-73fd-b22e-a8ebfbcd53e6/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"019f916c-2da4-7352-ace0-5e464338121a/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -658,7 +633,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert replacement, noted public bucket only affects reads, kept public-read/RLS intact, and added authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + "judgeNotes": "Diagnosed missing storage.objects UPDATE policy for upsert replacement, kept public-read bucket/RLS intact, and added authenticated owner-scoped UPDATE policy with USING and WITH CHECK." } ], "skills": { @@ -717,17 +692,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"4cbcfda6-5bde-4706-aadc-d0c3d58f1c19\",\"metric\":\"steps_a_mrxmh45d\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"8ce12f9c-b3b0-4900-8599-eb1bb0801635\",\"metric\":\"steps_a_mry65xqk\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"4cbcfda6-5bde-4706-aadc-d0c3d58f1c19\",\"metric\":\"steps_a_mrxmh45d\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"8ce12f9c-b3b0-4900-8599-eb1bb0801635\",\"metric\":\"steps_a_mry65xqk\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"45751830-2003-4463-bb10-512c6c1527da\",\"metric\":\"steps_b_mrxmh45d\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"cf5c096b-8d16-44ff-86be-7e9a375bc782\",\"metric\":\"steps_b_mry65xqk\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -755,150 +730,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server withSupabase edge function\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - } - ], - "resultChars": 84300 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key edge function apikey header\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - } - ], - "resultChars": 172535 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server reference withSupabase auth mode service admin client requireUser optional\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nuxt-3", - "title": "Build a User Management App with Nuxt 3" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - } - ], - "resultChars": 12 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nImplement it with the **`@supabase/server`** package, which is built for exactly\nthis kind of multi-auth Edge Function. Import it directly in your function:\n\n```ts\nimport { withSupabase } from \"npm:@supabase/server\";\n```\n\nOur product stores per-user metrics in a `user_stats` table that already exists\n(see `supabase/migrations/`), protected by row-level security so a user can read\nonly their own rows.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.\n\nGet the local stack running so the function is reachable at the path above.", "promptSourcePath": "evals/build-functions-006-dual-auth-with-server/PROMPT.md", @@ -934,7 +766,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The answer correctly treats the task as Supabase Realtime/Postgres Changes, adds the messages table to the supabase_realtime publication, and provides a postgres_changes subscription example. It does not recommend or imply read replicas are needed/useful for live updates, nor confuse them with logical replication/publications." + "judgeNotes": "The assistant correctly handled the request as Supabase Realtime/Postgres Changes setup by adding public.messages to the supabase_realtime publication and showing a postgres_changes subscription. It did not recommend or imply read replicas are needed/useful, and did not confuse read replicas with logical replication/publications." } ], "skills": { @@ -972,17 +804,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly recognized HTTP 546 worker/resource/CPU limit errors, not 500/503 server errors." + "judgeNotes": "The assistant correctly identified `video-thumbnails` as the affected function and recognized failures as HTTP 546 resource/CPU limit responses, not 500s or 503s." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence including `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)` and explicitly distinguishing it from wall-clock timeout." + "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing log evidence: shutdown reason CPUTime and cpu_time_used == cpu_time_limit == 2000ms. It also distinguishes this from wall-clock time, memory, crashes, and unrelated errors." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant explicitly recommends offloading thumbnail generation to a background worker/external container and reducing per-call CPU cost, rather than retrying or raising limits." + "judgeNotes": "Recommended offloading CPU-heavy thumbnail generation to a worker/background service and reducing CPU cost per call." } ], "skills": { @@ -1064,7 +896,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "Diagnoses missing table grants vs RLS, preserves owner-scoped RLS, grants SELECT/INSERT to authenticated only for client access, and keeps RLS enabled. Also recognizes secure-by-default/non-auto-exposed behavior implicitly via missing underlying PostgREST grants, without granting anon/public or weakening policies." + "judgeNotes": "Diagnoses missing Data API/table grants under secure-by-default behavior, distinguishes grants from RLS, preserves owner-scoped RLS, grants only SELECT/INSERT to authenticated, and does not weaken access." } ], "skills": { @@ -1107,7 +939,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"0857b42f-c7bb-4778-ba57-278fde808b35\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"6ac0ee7b-ee73-4a42-b5b0-060584208976\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -1116,7 +948,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "Diagnosed the silent zero-row UPDATE as an UPDATE RLS policy missing a USING clause, explained WITH CHECK vs USING, and fixed it with a policy scoped to authenticated users and user_id = auth.uid(), preserving WITH CHECK and RLS." + "judgeNotes": "Diagnoses the missing UPDATE USING clause on the tasks RLS policy, explains why WITH CHECK alone yields silent 0-row updates, and fixes it with USING (user_id = auth.uid()) while retaining WITH CHECK (user_id = auth.uid()) and authenticated/user-owned scoping." } ], "skills": { @@ -1127,31 +959,27 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"RLS UPDATE policy USING WITH CHECK missing USING clause\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"row level security policy UPDATE USING WITH CHECK default missing\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0024_permissive_rls_policy", - "title": "Database Advisor: Lint 0024_permissive_rls_policy" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", - "title": "RLS Simplified" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", + "title": "Build a User Management App with Ionic React" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/auth/enterprise-sso/auth-sso-saml", + "title": "Single Sign-On with SAML 2.0 for Projects" } ], - "resultChars": 67418 + "resultChars": 104495 } ] }, @@ -1183,7 +1011,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer clearly says pause/restore is not the right first recovery step, distinguishes restart from restore/pause, recommends restart plus logs/advisors and checking overload/sizing, and suggests contacting support if unable to restart. It is concrete and safe, though it could mention waiting/reducing workload more explicitly." + "judgeNotes": "Answers that restart, not pause/restore, is the first step; distinguishes restart from restore/pause; recommends restart and checking logs/advisors, with support as an option if unresolved. Minor concern: suggests pause/restore as a last resort, but not as the primary or reliable recovery path." } ], "skills": { @@ -1191,36 +1019,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"project unhealthy restart pause restore dashboard unusable\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" - }, - { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - } - ], - "resultChars": 28996 - } - ] + "calls": [] }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", "promptSourcePath": "evals/resolve-reliability-001-unhealthy-project-recovery/PROMPT.md", @@ -1268,7 +1067,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019f8f6a-de4a-72d5-b12a-29589a17a64c/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"019f916b-e43c-773b-bc26-811fe4472b5e/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -1277,7 +1076,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "The answer correctly diagnoses missing UPDATE RLS policy on storage.objects for Supabase Storage upsert, notes public bucket only affects reads, keeps public-read/RLS intact, and adds an authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + "judgeNotes": "Diagnoses missing UPDATE policy for upsert on storage.objects, distinguishes public bucket/read URL from RLS operations, adds authenticated owner-scoped UPDATE policy with USING and WITH CHECK, and keeps public read/RLS intact." } ], "skills": { From 88731b2a3b606bd377425a49e6a0800b048fe71d Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 01:20:40 +0100 Subject: [PATCH 9/9] feat: delete pair sandboxes after use; name them for the dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pair's VM is never reused: stop + delete so nothing lingers (the builder is deliberately kept — its snapshot is the warm-boot cache and is resolved by its name; verified deleting a from-snapshot sandbox leaves the source snapshot intact). Sandboxes are also named ---- — tags only show on the detail page, so the list view was unreadable random names; the suffix keeps retries and overlapping runs collision-free (names are project-unique). Co-Authored-By: Claude Fable 5 --- packages/vercel-runner/src/sandbox-job.ts | 26 +++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/vercel-runner/src/sandbox-job.ts b/packages/vercel-runner/src/sandbox-job.ts index 1eb32e49..769172e5 100644 --- a/packages/vercel-runner/src/sandbox-job.ts +++ b/packages/vercel-runner/src/sandbox-job.ts @@ -58,6 +58,20 @@ export interface SandboxJobOptions { resultsDir: string; } +/** + * Human-readable sandbox name for the dashboard list view (tags only show on + * the detail page): `----`. Names must be unique + * within the project, so a short random suffix keeps retries and repeat runs + * of the same pair from colliding. Sanitized to the accepted charset (e.g. + * "4.5" → "4-5") and length-capped defensively. + */ +function pairSandboxName(pair: EvalPair): string { + const clean = (value: string) => + value.toLowerCase().replaceAll(/[^a-z0-9-]+/g, "-"); + const suffix = Math.random().toString(36).slice(2, 6); + return `${clean(pair.experiment).slice(0, 40)}--${clean(pair.evalId).slice(0, 48)}--${suffix}`; +} + export interface SandboxJobResult { pair: EvalPair; ok: boolean; @@ -88,6 +102,7 @@ export async function runPairInSandbox( options.onPhase?.("create"); sandbox = await createSandbox( { + name: pairSandboxName(pair), resources: { vcpus: options.vcpus }, timeout: options.sandboxTimeoutMs, // Sandboxes auto-snapshot on stop by default ("persistent"); a @@ -247,12 +262,19 @@ export async function runPairInSandbox( }; } finally { if (sandbox) { + // A pair's VM is never reused: stop it and delete the record outright + // so nothing lingers (persistent:false already prevents auto-snapshot + // storage; deletion is belt-and-braces and keeps the dashboard clean). + // The snapshot builder's sandbox is deliberately NOT deleted — its + // snapshot is the warm-boot cache and Snapshot.list resolves it by the + // builder's name. try { await sandbox.stop(); - log("sandbox stopped"); + await sandbox.delete(); + log("sandbox stopped and deleted"); } catch (err) { stderr.write( - `sandbox stop failed (it will expire on its own): ${err instanceof Error ? err.message : String(err)}\n`, + `sandbox cleanup failed (it will expire on its own): ${err instanceof Error ? err.message : String(err)}\n`, ); } }